It continues to crash rapidly. Up to now crashed 4 times with different messages. You may want to read so report is attached.
Thanks, Valdrin ________________________________ From: Valdrin MUJA Sent: Wednesday, July 5, 2023 10:56 To: bugs@openbsd.org <bugs@openbsd.org> Subject: kernel diagnostic assertion "!_kernel_lock_held()" failed Hello, My system hanged with these error messages and gone to ddb. Here you can find the details. Also crash.txt includes more. OpenBSD TEST1 7.3 GENERIC.MP#1125 amd64 # sysctl kern.version kern.version=OpenBSD 7.3 (GENERIC.MP) #1125: Sat Mar 25 10:36:29 MDT 2023 dera...@amd64.openbsd.org:/usr/src/sys/arch/amd64/compile/GENERIC.MP # syspatch -l 001_bgpd 002_bgpd 003_rpki 004_ssl 005_libx11 ddb{3}> show panic *cpu0: uvm_fault(0xfffffd82067b5440, 0x8, 0, 1) -> e cpu3: kernel diagnostic assertion "!_kernel_lock_held()" failed: file "/usr/sr c/sys/kern/kern_fork.c", line 674 ddb{3}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 12883 18843K 18843K 78643K 36588 0 pcb 13 14K 18K 78643K 17 0 rtable 445 61K 73K 78643K 24030 0 ifaddr 233 52K 52K 78643K 1351 0 sysctl 3 1K 1K 78643K 3 0 counters 226 85K 85K 78643K 438 0 ioctlops 0 0K 4K 78643K 42936 0 iov 0 0K 4K 78643K 620 0 mount 8 8K 8K 78643K 9 0 log 0 0K 0K 78643K 1 0 vnodes 1220 77K 79K 78643K 4125 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 31 69K 69K 78643K 39 0 shm 2 1K 1K 78643K 2 0 VM map 2 1K 1K 78643K 2 0 sem 2 10K 10K 78643K 4 0 dirhash 156 29K 29K 78643K 198 0 ACPI 3556 426K 453K 78643K 13924 0 file desc 30 21K 24K 78643K 20846 0 sigio 1 0K 0K 78643K 2 0 proc 111 93K 101K 78643K 35667 0 MFS node 6 0K 0K 78643K 6 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 in_multi 37 2K 2K 78643K 180 0 ether_multi 2 0K 0K 78643K 3 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 31 148K 148K 78643K 31 0 exec 0 0K 1K 78643K 47003 0 pfkey data 0 0K 0K 78643K 2 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 1558K 78643K 16937 0 newblk 1 0K 0K 78643K 1 0 indirdep 0 0K 672K 78643K 1376 0 VM swap 8 1110K 1112K 78643K 10 0 UVM amap 4170 376K 379K 78643K 475831 0 UVM aobj 3 2K 2K 78643K 3 0 USB 11 10K 10K 78643K 13 0 USB device 4 0K 0K 78643K 4 0 USB HC 1 0K 0K 78643K 1 0 memdesc 1 4K 4K 78643K 1 0 crypto data 18 258K 258K 78643K 18 0 NDP 46 0K 0K 78643K 99 0 temp 202 5824K 5890K 78643K 1138487 0 kqueue 79 141K 147K 78643K 2026 0 SYN cache 2 16K 16K 78643K 2 0 ddb{3}> trace db_enter() at db_enter+0x10 panic(ffffffff82067518) at panic+0xbf __assert(ffffffff820de23b,ffffffff82121828,2a2,ffffffff8209881e) at __assert+0x 25 proc_trampoline_mp() at proc_trampoline_mp+0xcf end trace frame: 0x0, count: -4 ddb{3}> db_enter() at db_enter+0x10 end trace frame: 0xffff800022ab0d70, count: 0 ddb{3}> machine ddbcpu 0 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xa6 softintr_dispatch(0) at softintr_dispatch+0x49 Xsoftclock() at Xsoftclock+0x1f cnputc(62) at cnputc+0x37 db_putchar(62) at db_putchar+0x2ea kprintf() at kprintf+0x133b db_printf(ffffffff8211cc51) at db_printf+0x69 fault(ffffffff820cd05b) at fault+0x8e kpageflttrap(ffff800022a2cb60,8) at kpageflttrap+0x1cf kerntrap(ffff800022a2cb60) at kerntrap+0x91 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b end trace frame: 0xffff800022a2cc20, count: 0 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218c8ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 mtx_enter(ffff800001b79f18) at mtx_enter+0x48 msleep(ffff800001b79f00,ffff800001b79f18,20,ffffffff82099fe5,0) at msleep+0x12a taskq_next_work(ffff800001b79f00,ffff800022592090) at taskq_next_work+0x61 taskq_thread(ffff800001b79f00) at taskq_thread+0xeb end trace frame: 0x0, count: 8 ddb{1}> machine ddbcpu 2 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_release_all(ffffffff824bca80) at __mp_release_all+0x21 msleep(fffffd823727a730,fffffd823727a730,318,ffffffff820d6b1b,2) at msleep+0x10 b kqueue_sleep(fffffd823727a730,ffff800022264358) at kqueue_sleep+0xbe kqueue_scan(ffff800022264258,8,ffff800022264150,ffff800022264358,ffff8000fffe00 50,ffff8000222643ac) at kqueue_scan+0x108 sys_kevent(ffff8000fffe0050,ffff800022264410,ffff800022264470) at sys_kevent+0x 371 syscall(ffff8000222644e0) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x2b22a4b60, count: 5 ddb{2}> machine ddbcpu 3 Stopped at db_enter+0x10: popq %rbp db_enter() at db_enter+0x10 panic(ffffffff82067518) at panic+0xbf __assert(ffffffff820de23b,ffffffff82121828,2a2,ffffffff8209881e) at __assert+0x 25 proc_trampoline_mp() at proc_trampoline_mp+0xcf end trace frame: 0x0, count: 11 ddb{3}> machine ddbcpu 4 Invalid cpu 4 ddb{3}> machine ddbcpu 0t1 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218c8ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 mtx_enter(ffff800001b79f18) at mtx_enter+0x48 msleep(ffff800001b79f00,ffff800001b79f18,20,ffffffff82099fe5,0) at msleep+0x12a taskq_next_work(ffff800001b79f00,ffff800022592090) at taskq_next_work+0x61 taskq_thread(ffff800001b79f00) at taskq_thread+0xeb end trace frame: 0x0, count: 8 ddb{1}> machine ddbcpu 0t2 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_release_all(ffffffff824bca80) at __mp_release_all+0x21 msleep(fffffd823727a730,fffffd823727a730,318,ffffffff820d6b1b,2) at msleep+0x10 b kqueue_sleep(fffffd823727a730,ffff800022264358) at kqueue_sleep+0xbe kqueue_scan(ffff800022264258,8,ffff800022264150,ffff800022264358,ffff8000fffe00 50,ffff8000222643ac) at kqueue_scan+0x108 sys_kevent(ffff8000fffe0050,ffff800022264410,ffff800022264470) at sys_kevent+0x 371 syscall(ffff8000222644e0) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x2b22a4b60, count: 5 ddb{2}> machine ddbcpu 0t3 Stopped at db_enter+0x10: popq %rbp db_enter() at db_enter+0x10 panic(ffffffff82067518) at panic+0xbf __assert(ffffffff820de23b,ffffffff82121828,2a2,ffffffff8209881e) at __assert+0x 25 proc_trampoline_mp() at proc_trampoline_mp+0xcf end trace frame: 0x0, count: 11 ddb{3}> P.S. dmesg.txt is also attached.. -- Thanks for reading. Valdrin
CRASH 01 : ddb{3}> show panic *cpu0: uvm_fault(0xfffffd82067b5440, 0x8, 0, 1) -> e cpu3: kernel diagnostic assertion "!_kernel_lock_held()" failed: file "/usr/sr c/sys/kern/kern_fork.c", line 674 ddb{3}> ps PID TID PPID UID S FLAGS WAIT COMMAND *53409 454121 10474 0 7 0x100000 sh 58907 516606 10474 0 2 0x100000 sh 10474 417896 35677 0 2 0x100002 sh 52042 375502 89267 0 2 0x100002 ps 77764 83714 35677 0 3 0x10008a sigsusp sh 89267 222930 35677 0 3 0x100082 piperd sh 41409 870 35677 0 3 0x82 thrsleep mine-filter 41409 26909 35677 0 3 0x4000082 nanoslp mine-filter 41409 436637 35677 0 3 0x4000082 thrsleep mine-filter 41409 383490 35677 0 3 0x4000082 thrsleep mine-filter 41409 387156 35677 0 3 0x4000082 thrsleep mine-filter 41409 139392 35677 0 3 0x4000082 thrsleep mine-filter 41409 302614 35677 0 3 0x4000082 thrsleep mine-filter 41409 61846 35677 0 3 0x4000082 thrsleep mine-filter 41409 396396 35677 0 3 0x4000082 netio mine-filter 41409 77669 35677 0 3 0x4000082 thrsleep mine-filter 41409 432551 35677 0 3 0x4000082 thrsleep mine-filter 41409 305280 35677 0 3 0x4000082 netio mine-filter 41409 199867 35677 0 3 0x4000082 thrsleep mine-filter 41409 498811 35677 0 3 0x4000082 thrsleep mine-filter 41409 301417 35677 0 3 0x4000082 thrsleep mine-filter 41409 25438 35677 0 3 0x4000082 netio mine-filter 41409 188591 35677 0 3 0x4000082 thrsleep mine-filter 41409 66557 35677 0 3 0x4000082 thrsleep mine-filter 41409 247571 35677 0 3 0x4000082 thrsleep mine-filter 41409 480213 35677 0 3 0x4000082 kqread mine-filter 41409 496842 35677 0 3 0x4000082 thrsleep mine-filter 41409 352012 35677 0 3 0x4000082 thrsleep mine-filter 41409 49963 35677 0 3 0x4000082 thrsleep mine-filter 41409 324956 35677 0 3 0x4000082 thrsleep mine-filter 41409 49414 35677 0 3 0x4000082 netio mine-filter 41409 65635 35677 0 3 0x4000082 thrsleep mine-filter 26392 500122 62485 75 3 0x1100092 kqread bgpd 94740 458157 62485 75 3 0x1100092 kqread bgpd 31711 437095 62485 75 3 0x1100092 kqread bgpd 62485 48056 1 0 7 0 bgpd 38680 483276 34098 0 3 0x83 thrsleep minesh 38680 278236 34098 0 3 0x4000083 thrsleep minesh 38680 127964 34098 0 3 0x4000083 thrsleep minesh 38680 376298 34098 0 3 0x4000083 thrsleep minesh 38680 165768 34098 0 3 0x4000083 thrsleep minesh 38680 370077 34098 0 3 0x4000083 kqread minesh 38680 476528 34098 0 3 0x4000083 thrsleep minesh 38680 223917 34098 0 3 0x4000083 thrsleep minesh 38680 83255 34098 0 3 0x4000083 ttyin minesh 38680 145891 34098 0 3 0x4000083 thrsleep minesh 34098 391627 12941 0 3 0x10008b sigsusp ksh 12941 469320 1 0 3 0x100080 kqread tmux 71476 442894 1 0 3 0x82 thrsleep minesshd73 71476 25570 1 0 3 0x4000082 thrsleep minesshd73 71476 33137 1 0 3 0x4000082 thrsleep minesshd73 71476 14073 1 0 3 0x4000082 kqread minesshd73 71476 102328 1 0 3 0x4000082 thrsleep minesshd73 71476 97369 1 0 3 0x4000082 thrsleep minesshd73 71476 194464 1 0 3 0x4000082 thrsleep minesshd73 21723 459592 30444 67 3 0x90 kqread nginx 30444 230853 1 0 3 0x88 sigsusp nginx 76900 201770 35677 0 3 0x82 thrsleep mine-wan-health 76900 404375 35677 0 3 0x4000082 thrsleep mine-wan-health 76900 174219 35677 0 3 0x4000082 thrsleep mine-wan-health 76900 373529 35677 0 3 0x4000082 thrsleep mine-wan-health 76900 4377 35677 0 3 0x4000082 thrsleep mine-wan-health 76900 491455 35677 0 3 0x4000082 thrsleep mine-wan-health 76900 138577 35677 0 3 0x4000082 thrsleep mine-wan-health 76900 147944 35677 0 3 0x4000082 kqread mine-wan-health 26372 503310 35677 0 3 0x82 thrsleep mine-alarmd 26372 443895 35677 0 3 0x4000082 thrsleep mine-alarmd 26372 464332 35677 0 3 0x4000082 kqread mine-alarmd 26372 134881 35677 0 3 0x4000082 thrsleep mine-alarmd 26372 34808 35677 0 3 0x4000082 thrsleep mine-alarmd 26372 68007 35677 0 3 0x4000082 thrsleep mine-alarmd 26372 304746 35677 0 3 0x4000082 thrsleep mine-alarmd 52103 14397 35677 0 3 0x82 thrsleep mine-core-health che 52103 349746 35677 0 3 0x4000082 nanoslp mine-core-health che 52103 412063 35677 0 3 0x4000082 thrsleep mine-core-health che 52103 475177 35677 0 2 0x4000002 mine-core-health che 52103 460879 35677 0 3 0x4000082 thrsleep mine-core-health che 52103 92085 35677 0 3 0x4000082 thrsleep mine-core-health che 52103 26178 35677 0 3 0x4000082 thrsleep mine-core-health che 52103 1125 35677 0 3 0x4000082 thrsleep mine-core-health che 56789 386802 35677 0 3 0x82 kqread mine-bandwidth 56789 265559 35677 0 3 0x4000082 thrsleep mine-bandwidth 56789 184046 35677 0 3 0x4000082 thrsleep mine-bandwidth 56789 67005 35677 0 3 0x4000082 thrsleep mine-bandwidth 56789 19080 35677 0 3 0x4000082 thrsleep mine-bandwidth 56789 470207 35677 0 3 0x4000082 thrsleep mine-bandwidth 54522 193990 35677 0 3 0x82 kqread mine-siem 54522 425862 35677 0 3 0x4000082 thrsleep mine-siem 54522 331269 35677 0 3 0x4000082 thrsleep mine-siem 54522 205830 35677 0 3 0x4000082 thrsleep mine-siem 54522 224655 35677 0 3 0x4000082 thrsleep mine-siem 54522 292926 35677 0 3 0x4000082 thrsleep mine-siem 54522 306662 35677 0 3 0x4000082 thrsleep mine-siem 55971 307381 35677 0 3 0x82 kqread mine-make-pdf 55971 262056 35677 0 3 0x4000082 thrsleep mine-make-pdf 55971 447966 35677 0 3 0x4000082 thrsleep mine-make-pdf 55971 209150 35677 0 3 0x4000082 thrsleep mine-make-pdf 55971 381650 35677 0 3 0x4000082 thrsleep mine-make-pdf 55971 339221 35677 0 3 0x4000082 thrsleep mine-make-pdf 55971 236192 35677 0 3 0x4000082 thrsleep mine-make-pdf 90491 288981 35677 0 3 0x82 thrsleep mine-ips 90491 155316 35677 0 3 0x4000082 thrsleep mine-ips 90491 45012 35677 0 3 0x4000082 thrsleep mine-ips 90491 261871 35677 0 3 0x4000082 thrsleep mine-ips 90491 168569 35677 0 3 0x4000082 thrsleep mine-ips 90491 53098 35677 0 3 0x4000082 kqread mine-ips 90491 244240 35677 0 3 0x4000082 thrsleep mine-ips 29550 86948 1 109 3 0x90 kqread ftp-proxy 44043 101149 35677 0 3 0x82 thrsleep mine-telemetry 44043 298959 35677 0 3 0x4000082 thrsleep mine-telemetry 44043 462054 35677 0 3 0x4000082 thrsleep mine-telemetry 44043 139715 35677 0 3 0x4000082 thrsleep mine-telemetry 44043 122066 35677 0 3 0x4000082 thrsleep mine-telemetry 44043 363261 35677 0 3 0x4000082 thrsleep mine-telemetry 44043 309998 35677 0 3 0x4000082 thrsleep mine-telemetry 44043 437390 35677 0 3 0x4000082 thrsleep mine-telemetry 44043 64490 35677 0 3 0x4000082 kqread mine-telemetry 44043 312984 35677 0 3 0x4000082 thrsleep mine-telemetry 46983 263004 35677 0 3 0x82 thrsleep mine-hotspot 46983 111857 35677 0 3 0x4000082 thrsleep mine-hotspot 46983 491777 35677 0 3 0x4000082 thrsleep mine-hotspot 46983 282887 35677 0 3 0x4000082 kqread mine-hotspot 46983 36728 35677 0 3 0x4000082 thrsleep mine-hotspot 46983 434110 35677 0 3 0x4000082 thrsleep mine-hotspot 46983 170719 35677 0 3 0x4000082 thrsleep mine-hotspot 19670 280057 35677 0 3 0x82 thrsleep mine-sms-server 19670 451318 35677 0 3 0x4000082 thrsleep mine-sms-server 19670 516071 35677 0 3 0x4000082 thrsleep mine-sms-server 19670 355292 35677 0 3 0x4000082 kqread mine-sms-server 19670 385776 35677 0 3 0x4000082 thrsleep mine-sms-server 19670 508543 35677 0 3 0x4000082 thrsleep mine-sms-server 94941 384439 35677 0 3 0x82 kqread mine-actmonL7 94941 237887 35677 0 3 0x4000082 thrsleep mine-actmonL7 94941 276098 35677 0 3 0x4000082 thrsleep mine-actmonL7 94941 183600 35677 0 3 0x4000082 thrsleep mine-actmonL7 94941 5296 35677 0 3 0x4000082 thrsleep mine-actmonL7 94941 142492 35677 0 3 0x4000082 thrsleep mine-actmonL7 94941 430308 35677 0 3 0x4000082 thrsleep mine-actmonL7 81102 128162 35677 0 3 0x82 thrsleep mine-notifier 81102 127888 35677 0 3 0x4000082 thrsleep mine-notifier 81102 94240 35677 0 3 0x4000082 kqread mine-notifier 81102 440499 35677 0 3 0x4000082 thrsleep mine-notifier 81102 241102 35677 0 3 0x4000082 thrsleep mine-notifier 81102 197233 35677 0 3 0x4000082 thrsleep mine-notifier 81102 98579 35677 0 3 0x4000082 thrsleep mine-notifier 81102 181374 35677 0 3 0x4000082 thrsleep mine-notifier 48957 134520 35677 0 3 0x82 kqread mine-flt-helper 48957 360647 35677 0 3 0x4000082 thrsleep mine-flt-helper 48957 438668 35677 0 3 0x4000082 thrsleep mine-flt-helper 48957 26806 35677 0 3 0x4000082 thrsleep mine-flt-helper 48957 353886 35677 0 3 0x4000082 thrsleep mine-flt-helper 48957 272416 35677 0 3 0x4000082 thrsleep mine-flt-helper 48957 162349 35677 0 3 0x4000082 thrsleep mine-flt-helper 91580 311709 35677 0 3 0x82 thrsleep mine-pptmon 91580 132790 35677 0 3 0x4000082 thrsleep mine-pptmon 91580 221615 35677 0 3 0x4000082 kqread mine-pptmon 91580 220177 35677 0 3 0x4000082 thrsleep mine-pptmon 91580 73681 35677 0 3 0x4000082 thrsleep mine-pptmon 91580 124150 35677 0 3 0x4000082 thrsleep mine-pptmon 91580 173766 35677 0 3 0x4000082 thrsleep mine-pptmon 93392 45109 35677 0 3 0x82 thrsleep mine-helper 93392 103893 35677 0 3 0x4000082 thrsleep mine-helper 93392 475620 35677 0 3 0x4000082 thrsleep mine-helper 93392 198294 35677 0 3 0x4000082 thrsleep mine-helper 93392 166481 35677 0 3 0x4000082 thrsleep mine-helper 93392 279701 35677 0 3 0x4000082 thrsleep mine-helper 93392 476293 35677 0 3 0x4000082 kqread mine-helper 93392 96892 35677 0 3 0x4000082 thrsleep mine-helper 93392 192380 35677 0 3 0x4000082 thrsleep mine-helper 77986 107818 35677 0 3 0x82 kqread mine-micro 77986 376224 35677 0 3 0x4000082 thrsleep mine-micro 77986 264495 35677 0 3 0x4000082 thrsleep mine-micro 77986 479876 35677 0 3 0x4000082 thrsleep mine-micro 77986 419795 35677 0 3 0x4000082 thrsleep mine-micro 71642 403694 35677 0 3 0x82 thrsleep mine-macro 71642 384903 35677 0 3 0x4000082 thrsleep mine-macro 71642 425207 35677 0 3 0x4000082 thrsleep mine-macro 71642 441785 35677 0 3 0x4000082 thrsleep mine-macro 71642 187253 35677 0 3 0x4000082 thrsleep mine-macro 71642 78525 35677 0 3 0x4000082 thrsleep mine-macro 71642 452682 35677 0 3 0x4000082 thrsleep mine-macro 71642 66656 35677 0 3 0x4000082 thrsleep mine-macro 71642 422115 35677 0 3 0x4000082 kqread mine-macro 36097 122688 35677 0 3 0x82 thrsleep mine-msad 36097 70580 35677 0 3 0x4000082 thrsleep mine-msad 36097 154493 35677 0 3 0x4000082 thrsleep mine-msad 36097 416620 35677 0 3 0x4000082 thrsleep mine-msad 36097 516865 35677 0 3 0x4000082 kqread mine-msad 36097 425500 35677 0 3 0x4000082 thrsleep mine-msad 36097 129706 35677 0 3 0x4000082 thrsleep mine-msad 69865 459920 88498 73 3 0x1100090 kqread syslogd 88498 401370 1 0 3 0x100082 netio syslogd 94362 140282 37428 0 3 0x82 kqread ttyd 37428 334295 35677 0 3 0x10008a sigsusp sh 76208 316854 24044 91 3 0x92 kqread snmpd_metrics 24044 320340 1 0 3 0x100080 kqread snmpd 26816 44984 1 91 3 0x1100092 kqread snmpd 61151 176999 35677 0 3 0x82 thrsleep mine-today-analy tic 61151 447874 35677 0 3 0x4000082 thrsleep mine-today-analy tic 61151 501341 35677 0 3 0x4000082 thrsleep mine-today-analy tic 61151 474997 35677 0 3 0x4000082 thrsleep mine-today-analy tic 61151 187885 35677 0 3 0x4000082 kqread mine-today-analy tic 61151 417546 35677 0 3 0x4000082 thrsleep mine-today-analy tic 61151 69243 35677 0 3 0x4000082 thrsleep mine-today-analy tic 61151 217429 35677 0 3 0x4000082 thrsleep mine-today-analy tic 72620 43920 3032 74 3 0x1100092 bpf pflogd 3032 330435 1 0 3 0x80 netio pflogd 48816 313640 0 0 7 0x14200 wg_crypt 2686 112494 0 0 3 0x14200 bored wg_crypt 53490 33827 0 0 3 0x14200 bored wg_crypt 70796 350782 0 0 3 0x14200 bored wg_crypt 92476 89471 0 0 3 0x14200 bored wg_handshake 55192 272102 0 0 3 0x14200 bored wg_handshake 86545 402133 69249 0 3 0x82 kqread mine-ssl-proxy 86545 54369 69249 0 3 0x4000082 thrsleep mine-ssl-proxy 86545 516582 69249 0 3 0x4000082 thrsleep mine-ssl-proxy 86545 127393 69249 0 3 0x4000082 thrsleep mine-ssl-proxy 86545 102064 69249 0 3 0x4000082 thrsleep mine-ssl-proxy 86545 500515 69249 0 3 0x4000082 thrsleep mine-ssl-proxy 86545 144775 69249 0 3 0x4000082 thrsleep mine-ssl-proxy 69249 185788 35677 0 3 0x10008a sigsusp sh 70450 29936 56975 0 3 0x82 kqread redis-server 70450 192614 56975 0 3 0x4000082 fsleep redis-server 70450 185074 56975 0 3 0x4000082 fsleep redis-server 70450 255500 56975 0 3 0x4000082 fsleep redis-server 56975 314509 35677 0 3 0x10008a sigsusp sh 75982 469991 75440 502 3 0x92 kqread mariadbd 75982 33351 75440 502 3 0x4000092 fsleep mariadbd 75982 330286 75440 502 3 0x4000092 fsleep mariadbd 75982 374476 75440 502 3 0x4000092 fsleep mariadbd 75982 495191 75440 502 3 0x4000092 fsleep mariadbd 75982 488772 75440 502 3 0x4000092 fsleep mariadbd 75982 498925 75440 502 3 0x400009a sigwait mariadbd 75982 294554 75440 502 3 0x4000092 kqread mariadbd 75982 262511 75440 502 3 0x4000092 kqread mariadbd 75982 294730 75440 502 3 0x4000092 kqread mariadbd 75982 288099 75440 502 3 0x4000092 kqread mariadbd 75982 257010 75440 502 3 0x4000092 kqread mariadbd 75982 73761 75440 502 3 0x4000092 kqread mariadbd 75982 129897 75440 502 3 0x4000092 kqread mariadbd 75982 29065 75440 502 3 0x4000092 kqread mariadbd 75982 8357 75440 502 3 0x4000092 kqread mariadbd 75982 128685 75440 502 3 0x4000092 kqread mariadbd 75982 404937 75440 502 3 0x4000092 fsleep mariadbd 75982 521599 75440 502 3 0x4000092 kqread mariadbd 75982 221257 75440 502 3 0x4000092 fsleep mariadbd 75982 16484 75440 502 3 0x4000092 fsleep mariadbd 75982 222624 75440 502 3 0x4000092 fsleep mariadbd 75982 271704 75440 502 3 0x4000092 fsleep mariadbd 75982 308779 75440 502 3 0x4000092 fsleep mariadbd 75982 349801 75440 502 3 0x4000092 fsleep mariadbd 75982 479106 75440 502 3 0x4000092 fsleep mariadbd 75440 365729 1 0 3 0x10008a sigsusp sh 69109 294197 1 0 3 0x100098 kqread cron 90264 120852 1 797 3 0x100082 piperd logger 5635 374561 1 797 3 0x82 thrsleep influxd 5635 212975 1 797 3 0x4000082 thrsleep influxd 5635 336817 1 797 3 0x4000082 thrsleep influxd 5635 170880 1 797 3 0x4000082 kqread influxd 5635 285826 1 797 3 0x4000082 thrsleep influxd 5635 400810 1 797 3 0x4000082 thrsleep influxd 5635 33496 1 797 3 0x4000082 thrsleep influxd 5635 353278 1 797 3 0x4000082 thrsleep influxd 5635 256862 1 797 3 0x4000082 thrsleep influxd 5635 131562 1 797 3 0x4000082 thrsleep influxd 77805 399770 91118 67 3 0x90 netcon php-fpm-7.4 75007 146604 91118 67 3 0x90 netcon php-fpm-7.4 91118 521027 1 0 3 0x80 kqread php-fpm-7.4 76376 346633 78526 0 3 0x83 thrsleep minesh 76376 272425 78526 0 3 0x4000083 thrsleep minesh 76376 493654 78526 0 3 0x4000083 thrsleep minesh 76376 465326 78526 0 3 0x4000083 thrsleep minesh 76376 227926 78526 0 3 0x4000083 ttyin minesh 76376 442413 78526 0 3 0x4000083 thrsleep minesh 76376 278721 78526 0 3 0x4000083 thrsleep minesh 76376 339336 78526 0 3 0x4000083 kqread minesh 76376 284848 78526 0 3 0x4000083 thrsleep minesh 76376 58834 78526 0 3 0x4000083 thrsleep minesh 35677 419724 20831 0 3 0x82 wait mine-core 35677 441780 20831 0 3 0x4000082 nanoslp mine-core 35677 142174 20831 0 3 0x4000082 wait mine-core 35677 478480 20831 0 3 0x4000082 thrsleep mine-core 35677 296393 20831 0 3 0x4000082 thrsleep mine-core 35677 290959 20831 0 3 0x4000082 thrsleep mine-core 35677 295712 20831 0 3 0x4000082 wait mine-core 35677 33782 20831 0 3 0x4000082 wait mine-core 35677 79298 20831 0 3 0x4000082 wait mine-core 35677 120769 20831 0 3 0x4000082 wait mine-core 35677 455767 20831 0 3 0x4000082 wait mine-core 35677 523365 20831 0 7 0x4000002 mine-core 35677 429698 20831 0 3 0x4000082 wait mine-core 35677 101500 20831 0 3 0x4000082 wait mine-core 35677 316524 20831 0 3 0x4000082 wait mine-core 35677 332836 20831 0 3 0x4000082 wait mine-core 35677 22227 20831 0 3 0x4000082 wait mine-core 35677 437092 20831 0 3 0x4000082 wait mine-core 35677 43136 20831 0 3 0x4000082 wait mine-core 35677 33230 20831 0 3 0x4000082 wait mine-core 35677 365898 20831 0 3 0x4000082 wait mine-core 35677 118793 20831 0 3 0x4000082 wait mine-core 35677 92481 20831 0 3 0x4000082 wait mine-core 35677 306937 20831 0 3 0x4000082 wait mine-core 35677 411527 20831 0 3 0x4000082 wait mine-core 35677 25724 20831 0 3 0x4000082 wait mine-core 35677 134272 20831 0 3 0x4000082 wait mine-core 35677 231350 20831 0 3 0x4000082 thrsleep mine-core 35677 158485 20831 0 3 0x4000082 wait mine-core 35677 515481 20831 0 3 0x4000082 wait mine-core 35677 178524 20831 0 3 0x4000082 thrsleep mine-core 35677 330288 20831 0 3 0x4000082 wait mine-core 35677 47033 20831 0 3 0x4000082 thrsleep mine-core 35677 338013 20831 0 3 0x4000082 wait mine-core 35677 345804 20831 0 3 0x4000082 thrsleep mine-core 35677 170128 20831 0 3 0x4000082 thrsleep mine-core 20831 136546 1723 0 3 0x10008a sigsusp sh 1723 78628 1 0 3 0x80 wait tcd 71728 208745 1 0 3 0x80 mfsidl mount_mfs 70059 317080 0 0 3 0x14200 bored srdis 78526 452701 74136 0 3 0x10008b sigsusp sh 90821 194504 1 99 3 0x1100090 kqread sndiod 37723 511980 1 110 3 0x100090 kqread sndiod 96045 488449 1 0 3 0x88 kqread sshd 6907 407668 70926 77 3 0x100092 kqread dhcpleased 85709 156918 70926 77 3 0x100092 kqread dhcpleased 70926 97302 1 0 3 0x80 kqread dhcpleased 87037 274836 24805 115 3 0x100092 kqread slaacd 37205 253965 24805 115 3 0x100092 kqread slaacd 24805 458014 1 0 3 0x100080 kqread slaacd 80910 71151 1 0 3 0x80 mfsidl mount_mfs 97488 272313 1 0 3 0x80 mfsidl mount_mfs 56394 58531 1 0 3 0x80 mfsidl mount_mfs 54474 409239 1 0 3 0x80 mfsidl mount_mfs 74136 260831 1 0 3 0x10008b sigsusp sh 74803 292052 0 0 3 0x14200 bored smr 32967 428425 0 0 2 0x14200 zerothread 85676 459738 0 0 3 0x14200 aiodoned aiodoned 3977 241155 0 0 3 0x14200 syncer update 24566 186529 0 0 3 0x14200 cleaner cleaner 54255 155427 0 0 3 0x14200 reaper reaper 90962 185765 0 0 3 0x14200 pgdaemon pagedaemon 29380 222709 0 0 3 0x14200 mmctsk sdmmc0 32359 180296 0 0 3 0x14200 usbtsk usbtask 48460 11228 0 0 3 0x14200 usbatsk usbatsk 90494 73299 0 0 3 0x40014200 acpi0 acpi0 45562 105376 0 0 3 0x40014200 idle3 89925 447854 0 0 3 0x40014200 idle2 83426 382000 0 0 3 0x40014200 idle1 7693 360036 0 0 3 0x14200 bored sensors 86534 8444 0 0 2 0x14200 softnet 70698 200098 0 0 3 0x14200 bored softnet 1033 449697 0 0 3 0x14200 bored softnet 58177 473601 0 0 2 0x14200 softnet 6117 379028 0 0 3 0x14200 bored systqmp 65246 342284 0 0 3 0x14200 bored systq 89502 121087 0 0 3 0x40014200 bored softclock 69163 404148 0 0 3 0x40014200 idle0 1 57202 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{3}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 12883 18843K 18843K 78643K 36588 0 pcb 13 14K 18K 78643K 17 0 rtable 445 61K 73K 78643K 24030 0 ifaddr 233 52K 52K 78643K 1351 0 sysctl 3 1K 1K 78643K 3 0 counters 226 85K 85K 78643K 438 0 ioctlops 0 0K 4K 78643K 42936 0 iov 0 0K 4K 78643K 620 0 mount 8 8K 8K 78643K 9 0 log 0 0K 0K 78643K 1 0 vnodes 1220 77K 79K 78643K 4125 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 31 69K 69K 78643K 39 0 shm 2 1K 1K 78643K 2 0 VM map 2 1K 1K 78643K 2 0 sem 2 10K 10K 78643K 4 0 dirhash 156 29K 29K 78643K 198 0 ACPI 3556 426K 453K 78643K 13924 0 file desc 30 21K 24K 78643K 20846 0 sigio 1 0K 0K 78643K 2 0 proc 111 93K 101K 78643K 35667 0 MFS node 6 0K 0K 78643K 6 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 in_multi 37 2K 2K 78643K 180 0 ether_multi 2 0K 0K 78643K 3 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 31 148K 148K 78643K 31 0 exec 0 0K 1K 78643K 47003 0 pfkey data 0 0K 0K 78643K 2 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 1558K 78643K 16937 0 newblk 1 0K 0K 78643K 1 0 indirdep 0 0K 672K 78643K 1376 0 VM swap 8 1110K 1112K 78643K 10 0 UVM amap 4170 376K 379K 78643K 475831 0 UVM aobj 3 2K 2K 78643K 3 0 USB 11 10K 10K 78643K 13 0 USB device 4 0K 0K 78643K 4 0 USB HC 1 0K 0K 78643K 1 0 memdesc 1 4K 4K 78643K 1 0 crypto data 18 258K 258K 78643K 18 0 NDP 46 0K 0K 78643K 99 0 temp 202 5824K 5890K 78643K 1138487 0 kqueue 79 141K 147K 78643K 2026 0 SYN cache 2 16K 16K 78643K 2 0 ddb{3}> trace db_enter() at db_enter+0x10 panic(ffffffff82067518) at panic+0xbf __assert(ffffffff820de23b,ffffffff82121828,2a2,ffffffff8209881e) at __assert+0x 25 proc_trampoline_mp() at proc_trampoline_mp+0xcf end trace frame: 0x0, count: -4 ddb{3}> db_enter() at db_enter+0x10 end trace frame: 0xffff800022ab0d70, count: 0 ddb{3}> machine ddbcpu 0 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xa6 softintr_dispatch(0) at softintr_dispatch+0x49 Xsoftclock() at Xsoftclock+0x1f cnputc(62) at cnputc+0x37 db_putchar(62) at db_putchar+0x2ea kprintf() at kprintf+0x133b db_printf(ffffffff8211cc51) at db_printf+0x69 fault(ffffffff820cd05b) at fault+0x8e kpageflttrap(ffff800022a2cb60,8) at kpageflttrap+0x1cf kerntrap(ffff800022a2cb60) at kerntrap+0x91 alltraps_kern_meltdown() at alltraps_kern_meltdown+0x7b end trace frame: 0xffff800022a2cc20, count: 0 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218c8ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 mtx_enter(ffff800001b79f18) at mtx_enter+0x48 msleep(ffff800001b79f00,ffff800001b79f18,20,ffffffff82099fe5,0) at msleep+0x12a taskq_next_work(ffff800001b79f00,ffff800022592090) at taskq_next_work+0x61 taskq_thread(ffff800001b79f00) at taskq_thread+0xeb end trace frame: 0x0, count: 8 ddb{1}> machine ddbcpu 2 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_release_all(ffffffff824bca80) at __mp_release_all+0x21 msleep(fffffd823727a730,fffffd823727a730,318,ffffffff820d6b1b,2) at msleep+0x10 b kqueue_sleep(fffffd823727a730,ffff800022264358) at kqueue_sleep+0xbe kqueue_scan(ffff800022264258,8,ffff800022264150,ffff800022264358,ffff8000fffe00 50,ffff8000222643ac) at kqueue_scan+0x108 sys_kevent(ffff8000fffe0050,ffff800022264410,ffff800022264470) at sys_kevent+0x 371 syscall(ffff8000222644e0) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x2b22a4b60, count: 5 ddb{2}> machine ddbcpu 3 Stopped at db_enter+0x10: popq %rbp db_enter() at db_enter+0x10 panic(ffffffff82067518) at panic+0xbf __assert(ffffffff820de23b,ffffffff82121828,2a2,ffffffff8209881e) at __assert+0x 25 proc_trampoline_mp() at proc_trampoline_mp+0xcf end trace frame: 0x0, count: 11 ddb{3}> machine ddbcpu 4 Invalid cpu 4 ddb{3}> machine ddbcpu 0t1 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218c8ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 mtx_enter(ffff800001b79f18) at mtx_enter+0x48 msleep(ffff800001b79f00,ffff800001b79f18,20,ffffffff82099fe5,0) at msleep+0x12a taskq_next_work(ffff800001b79f00,ffff800022592090) at taskq_next_work+0x61 taskq_thread(ffff800001b79f00) at taskq_thread+0xeb end trace frame: 0x0, count: 8 ddb{1}> machine ddbcpu 0t2 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_release_all(ffffffff824bca80) at __mp_release_all+0x21 msleep(fffffd823727a730,fffffd823727a730,318,ffffffff820d6b1b,2) at msleep+0x10 b kqueue_sleep(fffffd823727a730,ffff800022264358) at kqueue_sleep+0xbe kqueue_scan(ffff800022264258,8,ffff800022264150,ffff800022264358,ffff8000fffe00 50,ffff8000222643ac) at kqueue_scan+0x108 sys_kevent(ffff8000fffe0050,ffff800022264410,ffff800022264470) at sys_kevent+0x 371 syscall(ffff8000222644e0) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x2b22a4b60, count: 5 ddb{2}> machine ddbcpu 0t3 Stopped at db_enter+0x10: popq %rbp db_enter() at db_enter+0x10 panic(ffffffff82067518) at panic+0xbf __assert(ffffffff820de23b,ffffffff82121828,2a2,ffffffff8209881e) at __assert+0x 25 proc_trampoline_mp() at proc_trampoline_mp+0xcf end trace frame: 0x0, count: 11 ddb{3}> ---------------------------------------------------------------------------------- CRASH 02 : ddb{3}> show panic *cpu3: kernel diagnostic assertion "!ISSET(rt->rt_flags, RTF_UP)" failed: file " /usr/src/sys/net/route.c", line 496 ddb{3}> trace db_enter() at db_enter+0x10 panic(ffffffff82067518) at panic+0xbf __assert(ffffffff820de23b,ffffffff8206be5d,1f0,ffffffff820e901b) at __assert+0x 25 rtfree(fffffd8275365a90) at rtfree+0x1af route_output(fffffd8065dd1f00,fffffd821540a920) at route_output+0x413 route_send(fffffd821540a920,fffffd8065dd1f00,0,0) at route_send+0x57 sosend(fffffd821540a920,0,ffff80002254d3e0,0,0,80) at sosend+0x37f dofilewritev(ffff80002251f390,6,ffff80002254d3e0,0,ffff80002254d4e0) at dofilew ritev+0x14d sys_writev(ffff80002251f390,ffff80002254d480,ffff80002254d4e0) at sys_writev+0x d2 syscall(ffff80002254d550) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7699faee55e0, count: -11 ddb{3}> machine ddbcpu 0 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 msleep(ffffffff8253cc44,ffffffff8253cda8,7f,ffffffff820442c7,0) at msleep+0xe0 uvm_pagezero_thread(ffff8000ffff5b40) at uvm_pagezero_thread+0xa6 end trace frame: 0x0, count: 10 ddb{0}> machine ddbcpu 1 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218c8ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xb2 reaper(ffff8000ffff8008) at reaper+0x133 end trace frame: 0x0, count: 10 ddb{1}> machine ddbcpu 2 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xb2 rtfree(fffffd8275365c50) at rtfree+0x4c ip_output(fffffd8002a23200,0,fffffd81e1b055d0,800,0,fffffd81e1b05558,89c3b2da6f 974957) at ip_output+0x218 tcp_output(ffff800001c863c8) at tcp_output+0x168e tcp_send(fffffd81623d1a28,fffffd8075758600,0,0) at tcp_send+0x84 sosend(fffffd81623d1a28,0,ffff80002252b6b8,0,0,80) at sosend+0x37f sendit(ffff8000224deb48,9,ffff80002252b820,0,ffff80002252b910) at sendit+0x373 sys_sendmsg(ffff8000224deb48,ffff80002252b8b0,ffff80002252b910) at sys_sendmsg+ 0x131 syscall(ffff80002252b980) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7ce33fafee50, count: 2 ddb{2}> machine ddbcpu 3 Stopped at db_enter+0x10: popq %rbp db_enter() at db_enter+0x10 panic(ffffffff82067518) at panic+0xbf __assert(ffffffff820de23b,ffffffff8206be5d,1f0,ffffffff820e901b) at __assert+0x 25 rtfree(fffffd8275365a90) at rtfree+0x1af route_output(fffffd8065dd1f00,fffffd821540a920) at route_output+0x413 route_send(fffffd821540a920,fffffd8065dd1f00,0,0) at route_send+0x57 sosend(fffffd821540a920,0,ffff80002254d3e0,0,0,80) at sosend+0x37f dofilewritev(ffff80002251f390,6,ffff80002254d3e0,0,ffff80002254d4e0) at dofilew ritev+0x14d sys_writev(ffff80002251f390,ffff80002254d480,ffff80002254d4e0) at sys_writev+0x d2 syscall(ffff80002254d550) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7699faee55e0, count: 4 ddb{3}> machine ddbcpu 4 Invalid cpu 4 ddb{3}> machine ddbcpu 0t0 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 msleep(ffffffff8253cc44,ffffffff8253cda8,7f,ffffffff820442c7,0) at msleep+0xe0 uvm_pagezero_thread(ffff8000ffff5b40) at uvm_pagezero_thread+0xa6 end trace frame: 0x0, count: 10 ddb{0}> machine ddbcpu 0t1 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218c8ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xb2 reaper(ffff8000ffff8008) at reaper+0x133 end trace frame: 0x0, count: 10 ddb{1}> machine ddbcpu 0t2 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xb2 rtfree(fffffd8275365c50) at rtfree+0x4c ip_output(fffffd8002a23200,0,fffffd81e1b055d0,800,0,fffffd81e1b05558,89c3b2da6f 974957) at ip_output+0x218 tcp_output(ffff800001c863c8) at tcp_output+0x168e tcp_send(fffffd81623d1a28,fffffd8075758600,0,0) at tcp_send+0x84 sosend(fffffd81623d1a28,0,ffff80002252b6b8,0,0,80) at sosend+0x37f sendit(ffff8000224deb48,9,ffff80002252b820,0,ffff80002252b910) at sendit+0x373 sys_sendmsg(ffff8000224deb48,ffff80002252b8b0,ffff80002252b910) at sys_sendmsg+ 0x131 syscall(ffff80002252b980) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7ce33fafee50, count: 2 ddb{2}> machine ddbcpu 0t3 Stopped at db_enter+0x10: popq %rbp db_enter() at db_enter+0x10 panic(ffffffff82067518) at panic+0xbf __assert(ffffffff820de23b,ffffffff8206be5d,1f0,ffffffff820e901b) at __assert+0x 25 rtfree(fffffd8275365a90) at rtfree+0x1af route_output(fffffd8065dd1f00,fffffd821540a920) at route_output+0x413 route_send(fffffd821540a920,fffffd8065dd1f00,0,0) at route_send+0x57 sosend(fffffd821540a920,0,ffff80002254d3e0,0,0,80) at sosend+0x37f dofilewritev(ffff80002251f390,6,ffff80002254d3e0,0,ffff80002254d4e0) at dofilew ritev+0x14d sys_writev(ffff80002251f390,ffff80002254d480,ffff80002254d4e0) at sys_writev+0x d2 syscall(ffff80002254d550) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7699faee55e0, count: 4 ddb{3}> machine ddbcpu 0t4 Invalid cpu 4 ddb{3}> ---------------------------------------------------------------------------------- CRASH 03 : TEST1> uvm_fault(0xfffffd826717bcc0, 0x8, 0, 1) -> e kernel: page fault trap, code=0 Stopped at srp_get_locked+0x11: movq 0(%rdi),%rax TID PID UID PRFLAGS PFLAGS CPU COMMAND *225335 47125 0 0 0 1 bgpd 231752 78299 73 0x1100010 0 3 syslogd 344909 6421 0 0x14000 0x200 2 wg_handshake 361415 98860 0 0x14000 0x200 0 reaper srp_get_locked(8) at srp_get_locked+0x11 rtable_mpath_reprio(0,ffff800007fb2f00,18,30,fffffd81e341f558) at rtable_mpath_ reprio+0x175 rt_if_linkstate_change(fffffd81e341f558,ffff800001b34000,0) at rt_if_linkstate_ change+0xcd rtm_output(ffff80000097f000,ffff8000225d54c0,ffff8000225d5418,30,0) at rtm_outp ut+0x71e route_output(fffffd8002aff600,fffffd81e22e4600) at route_output+0x3bc route_send(fffffd81e22e4600,fffffd8002aff600,0,0) at route_send+0x57 sosend(fffffd81e22e4600,0,ffff8000225d5750,0,0,80) at sosend+0x37f dofilewritev(ffff800022575bb8,6,ffff8000225d5750,0,ffff8000225d5850) at dofilew ritev+0x14d sys_writev(ffff800022575bb8,ffff8000225d57f0,ffff8000225d5850) at sys_writev+0x d2 syscall(ffff8000225d58c0) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7b583d6b8570, count: 4 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{1}> show panic *cpu1: uvm_fault(0xfffffd826717bcc0, 0x8, 0, 1) -> e ddb{1}> show malloc Type InUse MemUse HighUse Limit Requests Type Lim devbuf 12882 18843K 18843K 78643K 36398 0 pcb 13 14K 18K 78643K 19 0 rtable 473 63K 78K 78643K 3131 0 ifaddr 236 52K 53K 78643K 1429 0 sysctl 3 1K 1K 78643K 3 0 counters 226 85K 85K 78643K 226 0 ioctlops 0 0K 4K 78643K 9602 0 iov 0 0K 2K 78643K 78 0 mount 8 8K 8K 78643K 9 0 log 0 0K 0K 78643K 1 0 vnodes 1220 77K 79K 78643K 3833 0 UFS quota 1 32K 32K 78643K 1 0 UFS mount 31 69K 69K 78643K 39 0 shm 2 1K 1K 78643K 2 0 VM map 2 1K 1K 78643K 2 0 sem 2 10K 10K 78643K 4 0 dirhash 153 28K 28K 78643K 195 0 ACPI 3556 426K 453K 78643K 13924 0 file desc 29 21K 24K 78643K 2807 0 sigio 1 0K 0K 78643K 1 0 proc 101 76K 101K 78643K 6602 0 MFS node 6 0K 0K 78643K 6 0 NFS srvsock 1 0K 0K 78643K 1 0 NFS daemon 1 16K 16K 78643K 1 0 in_multi 37 2K 2K 78643K 48 0 ether_multi 2 0K 0K 78643K 2 0 ISOFS mount 1 32K 32K 78643K 1 0 MSDOSFS mount 1 16K 16K 78643K 1 0 ttys 25 122K 122K 78643K 25 0 exec 0 0K 1K 78643K 6746 0 pfkey data 0 0K 0K 78643K 2 0 tdb 3 0K 0K 78643K 3 0 pagedep 1 8K 8K 78643K 1 0 inodedep 1 32K 1047K 78643K 17614 0 newblk 1 0K 0K 78643K 1 0 indirdep 0 0K 256K 78643K 235 0 VM swap 8 1110K 1112K 78643K 10 0 UVM amap 3693 346K 348K 78643K 69709 0 UVM aobj 3 2K 2K 78643K 3 0 USB 11 10K 10K 78643K 13 0 USB device 4 0K 0K 78643K 4 0 USB HC 1 0K 0K 78643K 1 0 memdesc 1 4K 4K 78643K 1 0 crypto data 18 258K 258K 78643K 18 0 NDP 46 0K 0K 78643K 46 0 temp 200 5824K 5889K 78643K 145594 0 kqueue 70 125K 143K 78643K 288 0 SYN cache 2 16K 16K 78643K 2 0 ddb{1}> trace srp_get_locked(8) at srp_get_locked+0x11 rtable_mpath_reprio(0,ffff800007fb2f00,18,30,fffffd81e341f558) at rtable_mpath_ reprio+0x175 rt_if_linkstate_change(fffffd81e341f558,ffff800001b34000,0) at rt_if_linkstate_ change+0xcd rtm_output(ffff80000097f000,ffff8000225d54c0,ffff8000225d5418,30,0) at rtm_outp ut+0x71e route_output(fffffd8002aff600,fffffd81e22e4600) at route_output+0x3bc route_send(fffffd81e22e4600,fffffd8002aff600,0,0) at route_send+0x57 sosend(fffffd81e22e4600,0,ffff8000225d5750,0,0,80) at sosend+0x37f dofilewritev(ffff800022575bb8,6,ffff8000225d5750,0,ffff8000225d5850) at dofilew ritev+0x14d sys_writev(ffff800022575bb8,ffff8000225d57f0,ffff8000225d5850) at sys_writev+0x d2 syscall(ffff8000225d58c0) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7b583d6b8570, count: -11 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xb2 softintr_dispatch(0) at softintr_dispatch+0x49 Xsoftclock() at Xsoftclock+0x1f _kernel_lock() at _kernel_lock+0xb2 reaper(ffff8000ffff8da0) at reaper+0x133 end trace frame: 0x0, count: 7 ddb{0}> machine ddbcpu 1 Stopped at srp_get_locked+0x11: movq 0(%rdi),%rax srp_get_locked(8) at srp_get_locked+0x11 rtable_mpath_reprio(0,ffff800007fb2f00,18,30,fffffd81e341f558) at rtable_mpath_ reprio+0x175 rt_if_linkstate_change(fffffd81e341f558,ffff800001b34000,0) at rt_if_linkstate_ change+0xcd rtm_output(ffff80000097f000,ffff8000225d54c0,ffff8000225d5418,30,0) at rtm_outp ut+0x71e route_output(fffffd8002aff600,fffffd81e22e4600) at route_output+0x3bc route_send(fffffd81e22e4600,fffffd8002aff600,0,0) at route_send+0x57 sosend(fffffd81e22e4600,0,ffff8000225d5750,0,0,80) at sosend+0x37f dofilewritev(ffff800022575bb8,6,ffff8000225d5750,0,ffff8000225d5850) at dofilew ritev+0x14d sys_writev(ffff800022575bb8,ffff8000225d57f0,ffff8000225d5850) at sys_writev+0x d2 syscall(ffff8000225d58c0) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7b583d6b8570, count: 4 ddb{1}> machine ddbcpu 2 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xb2 ifa_ifwithaddr(ffff8000224ed160,0) at ifa_ifwithaddr+0x2e in_pcbaddrisavail(fffffd82101dc400,ffff8000224ed160,0,ffff8000224fb8e8) at in_p cbaddrisavail+0xca udp_output(fffffd82101dc400,fffffd8079b94c00,ffff8000224ed2a0,fffffd8074318e00) at udp_output+0x23e sosend(fffffd823147b760,ffff8000224ed2a0,0,fffffd8079b94c00,fffffd8074318e00,0) at sosend+0x37f wg_send(ffff800001afa000,ffff8000224ed468,fffffd8079b94c00) at wg_send+0x115 wg_send_buf(ffff800001afa000,ffff8000224ed468,ffff8000224ed4f8,94) at wg_send_b uf+0x6b wg_peer_send_buf(ffff80000801d838,ffff8000224ed4f8,94) at wg_peer_send_buf+0x10 7 wg_send_initiation(ffff80000801d838) at wg_send_initiation+0x13d taskq_thread(ffff800001af9480) at taskq_thread+0x100 end trace frame: 0x0, count: 2 ddb{2}> machine ddbcpu 3 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218daff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xb2 vn_write(fffffd81d35fa3c8,ffff8000226cd530,0) at vn_write+0x3b dofilewritev(ffff800022297070,e,ffff8000226cd530,0,ffff8000226cd630) at dofilew ritev+0x14d sys_writev(ffff800022297070,ffff8000226cd5d0,ffff8000226cd630) at sys_writev+0x d2 syscall(ffff8000226cd6a0) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f592e3e8d20, count: 6 ddb{3}> machine ddbcpu 4 Invalid cpu 4 ddb{3}> machine ddbcpu 0t0 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xb2 softintr_dispatch(0) at softintr_dispatch+0x49 Xsoftclock() at Xsoftclock+0x1f _kernel_lock() at _kernel_lock+0xb2 reaper(ffff8000ffff8da0) at reaper+0x133 end trace frame: 0x0, count: 7 ddb{0}> machine ddbcpu 0t1 Stopped at srp_get_locked+0x11: movq 0(%rdi),%rax srp_get_locked(8) at srp_get_locked+0x11 rtable_mpath_reprio(0,ffff800007fb2f00,18,30,fffffd81e341f558) at rtable_mpath_ reprio+0x175 rt_if_linkstate_change(fffffd81e341f558,ffff800001b34000,0) at rt_if_linkstate_ change+0xcd rtm_output(ffff80000097f000,ffff8000225d54c0,ffff8000225d5418,30,0) at rtm_outp ut+0x71e route_output(fffffd8002aff600,fffffd81e22e4600) at route_output+0x3bc route_send(fffffd81e22e4600,fffffd8002aff600,0,0) at route_send+0x57 sosend(fffffd81e22e4600,0,ffff8000225d5750,0,0,80) at sosend+0x37f dofilewritev(ffff800022575bb8,6,ffff8000225d5750,0,ffff8000225d5850) at dofilew ritev+0x14d sys_writev(ffff800022575bb8,ffff8000225d57f0,ffff8000225d5850) at sys_writev+0x d2 syscall(ffff8000225d58c0) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7b583d6b8570, count: 4 ddb{1}> machine ddbcpu 0t2 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xb2 ifa_ifwithaddr(ffff8000224ed160,0) at ifa_ifwithaddr+0x2e in_pcbaddrisavail(fffffd82101dc400,ffff8000224ed160,0,ffff8000224fb8e8) at in_p cbaddrisavail+0xca udp_output(fffffd82101dc400,fffffd8079b94c00,ffff8000224ed2a0,fffffd8074318e00) at udp_output+0x23e sosend(fffffd823147b760,ffff8000224ed2a0,0,fffffd8079b94c00,fffffd8074318e00,0) at sosend+0x37f wg_send(ffff800001afa000,ffff8000224ed468,fffffd8079b94c00) at wg_send+0x115 wg_send_buf(ffff800001afa000,ffff8000224ed468,ffff8000224ed4f8,94) at wg_send_b uf+0x6b wg_peer_send_buf(ffff80000801d838,ffff8000224ed4f8,94) at wg_peer_send_buf+0x10 7 wg_send_initiation(ffff80000801d838) at wg_send_initiation+0x13d taskq_thread(ffff800001af9480) at taskq_thread+0x100 end trace frame: 0x0, count: 2 ddb{2}> machine ddbcpu 0t3 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218daff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xb2 vn_write(fffffd81d35fa3c8,ffff8000226cd530,0) at vn_write+0x3b dofilewritev(ffff800022297070,e,ffff8000226cd530,0,ffff8000226cd630) at dofilew ritev+0x14d sys_writev(ffff800022297070,ffff8000226cd5d0,ffff8000226cd630) at sys_writev+0x d2 syscall(ffff8000226cd6a0) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f592e3e8d20, count: 6 ddb{3}> machine ddbcpu 0t4 Invalid cpu 4 ddb{3}> ---------------------------------------------------------------------------------- CRASH 04 : SPOKE1> uvm_fault(0xfffffd81d5995878, 0x8, 0, 1) -> e kernel: page fault trap, code=0 Stopped at srp_get_locked+0x11: movq 0(%rdi),%rax TID PID UID PRFLAGS PFLAGS CPU COMMAND 448769 98731 0 0x100002 0 3 sh 350289 69698 73 0x1100010 0 0 syslogd *114462 84824 0 0 0 1 bgpd 256495 50081 0 0x14000 0x200 2 wg_handshake srp_get_locked(8) at srp_get_locked+0x11 rtable_mpath_reprio(0,ffff800001ca0890,18,30,fffffd8276a30a28) at rtable_mpath_ reprio+0x175 rt_if_linkstate_change(fffffd8276a30a28,ffff800001bbc000,0) at rt_if_linkstate_ change+0xcd rtm_output(ffff800000a41100,ffff8000224dd130,ffff8000224dd088,30,0) at rtm_outp ut+0x71e route_output(fffffd8071fd1800,fffffd8213cfbcb0) at route_output+0x3bc route_send(fffffd8213cfbcb0,fffffd8071fd1800,0,0) at route_send+0x57 sosend(fffffd8213cfbcb0,0,ffff8000224dd3c0,0,0,80) at sosend+0x37f dofilewritev(ffff800022512318,6,ffff8000224dd3c0,0,ffff8000224dd4c0) at dofilew ritev+0x14d sys_writev(ffff800022512318,ffff8000224dd460,ffff8000224dd4c0) at sys_writev+0x d2 syscall(ffff8000224dd530) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7c334ef822e0, count: 4 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{1}> show panic *cpu1: uvm_fault(0xfffffd81d5995878, 0x8, 0, 1) -> e ddb{1}> trace srp_get_locked(8) at srp_get_locked+0x11 rtable_mpath_reprio(0,ffff800001ca0890,18,30,fffffd8276a30a28) at rtable_mpath_ reprio+0x175 rt_if_linkstate_change(fffffd8276a30a28,ffff800001bbc000,0) at rt_if_linkstate_ change+0xcd rtm_output(ffff800000a41100,ffff8000224dd130,ffff8000224dd088,30,0) at rtm_outp ut+0x71e route_output(fffffd8071fd1800,fffffd8213cfbcb0) at route_output+0x3bc route_send(fffffd8213cfbcb0,fffffd8071fd1800,0,0) at route_send+0x57 sosend(fffffd8213cfbcb0,0,ffff8000224dd3c0,0,0,80) at sosend+0x37f dofilewritev(ffff800022512318,6,ffff8000224dd3c0,0,ffff8000224dd4c0) at dofilew ritev+0x14d sys_writev(ffff800022512318,ffff8000224dd460,ffff8000224dd4c0) at sys_writev+0x d2 syscall(ffff8000224dd530) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7c334ef822e0, count: -11 ddb{1}> machine ddbcpu 0 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xa6 vn_write(fffffd82120ca790,ffff80002268db60,0) at vn_write+0x3b dofilewritev(ffff8000225658f0,12,ffff80002268db60,0,ffff80002268dc60) at dofile writev+0x14d sys_writev(ffff8000225658f0,ffff80002268dc00,ffff80002268dc60) at sys_writev+0x d2 syscall(ffff80002268dcd0) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x70d107adf4d0, count: 6 ddb{0}> machine ddbcpu 1 Stopped at srp_get_locked+0x11: movq 0(%rdi),%rax srp_get_locked(8) at srp_get_locked+0x11 rtable_mpath_reprio(0,ffff800001ca0890,18,30,fffffd8276a30a28) at rtable_mpath_ reprio+0x175 rt_if_linkstate_change(fffffd8276a30a28,ffff800001bbc000,0) at rt_if_linkstate_ change+0xcd rtm_output(ffff800000a41100,ffff8000224dd130,ffff8000224dd088,30,0) at rtm_outp ut+0x71e route_output(fffffd8071fd1800,fffffd8213cfbcb0) at route_output+0x3bc route_send(fffffd8213cfbcb0,fffffd8071fd1800,0,0) at route_send+0x57 sosend(fffffd8213cfbcb0,0,ffff8000224dd3c0,0,0,80) at sosend+0x37f dofilewritev(ffff800022512318,6,ffff8000224dd3c0,0,ffff8000224dd4c0) at dofilew ritev+0x14d sys_writev(ffff800022512318,ffff8000224dd460,ffff8000224dd4c0) at sys_writev+0x d2 syscall(ffff8000224dd530) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7c334ef822e0, count: 4 ddb{1}> machine ddbcpu 2 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 mtx_enter_try(ffff8000000c6c98) at mtx_enter_try+0x17 mtx_enter(ffff8000000c6c98) at mtx_enter+0x35 msleep(ffff8000000c6c80,ffff8000000c6c98,20,ffffffff82099fe5,0) at msleep+0x12a taskq_next_work(ffff8000000c6c80,ffff8000224e3440) at taskq_next_work+0x61 taskq_thread(ffff8000000c6c80) at taskq_thread+0xeb end trace frame: 0x0, count: 7 ddb{2}> machine ddbcpu 3 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218daff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xb2 syscall(ffff800022af5ce0) at syscall+0x39b Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x77c660727b70, count: 9 ddb{3}> machine ddbcpu 4 Invalid cpu 4 ddb{3}> machine ddbcpu 0t0 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffffffff8244bff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xa6 vn_write(fffffd82120ca790,ffff80002268db60,0) at vn_write+0x3b dofilewritev(ffff8000225658f0,12,ffff80002268db60,0,ffff80002268dc60) at dofile writev+0x14d sys_writev(ffff8000225658f0,ffff80002268dc00,ffff80002268dc60) at sys_writev+0x d2 syscall(ffff80002268dcd0) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x70d107adf4d0, count: 6 ddb{0}> machine ddbcpu 0t1 Stopped at srp_get_locked+0x11: movq 0(%rdi),%rax srp_get_locked(8) at srp_get_locked+0x11 rtable_mpath_reprio(0,ffff800001ca0890,18,30,fffffd8276a30a28) at rtable_mpath_ reprio+0x175 rt_if_linkstate_change(fffffd8276a30a28,ffff800001bbc000,0) at rt_if_linkstate_ change+0xcd rtm_output(ffff800000a41100,ffff8000224dd130,ffff8000224dd088,30,0) at rtm_outp ut+0x71e route_output(fffffd8071fd1800,fffffd8213cfbcb0) at route_output+0x3bc route_send(fffffd8213cfbcb0,fffffd8071fd1800,0,0) at route_send+0x57 sosend(fffffd8213cfbcb0,0,ffff8000224dd3c0,0,0,80) at sosend+0x37f dofilewritev(ffff800022512318,6,ffff8000224dd3c0,0,ffff8000224dd4c0) at dofilew ritev+0x14d sys_writev(ffff800022512318,ffff8000224dd460,ffff8000224dd4c0) at sys_writev+0x d2 syscall(ffff8000224dd530) at syscall+0x3d4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7c334ef822e0, count: 4 ddb{1}> machine ddbcpu 0t2 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 mtx_enter_try(ffff8000000c6c98) at mtx_enter_try+0x17 mtx_enter(ffff8000000c6c98) at mtx_enter+0x35 msleep(ffff8000000c6c80,ffff8000000c6c98,20,ffffffff82099fe5,0) at msleep+0x12a taskq_next_work(ffff8000000c6c80,ffff8000224e3440) at taskq_next_work+0x61 taskq_thread(ffff8000000c6c80) at taskq_thread+0xeb end trace frame: 0x0, count: 7 ddb{2}> machine ddbcpu 0t3 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff8000218daff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 _kernel_lock() at _kernel_lock+0xb2 syscall(ffff800022af5ce0) at syscall+0x39b Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x77c660727b70, count: 9 ddb{3}> machine ddbcpu 0t4 Invalid cpu 4 ddb{3}>