On Sat, Sep 20, 2025 at 10:59:51PM +0200, Claudio Jeker wrote:
> The M10-1 hits this panic in roughly 24h of running make -j 32 build in a
> loop. First time it exploded inside the reaper for me. So maybe this is
> closer to the truth.
Another run, took a bit more than 24h this time.
This is from my top running when it paniced:
load averages: 20.59, 10.40, 6.74 m10.zyd.ch 08:39:30
135 processes: 6 starting, 4 running, 98 idle, 1 dead, 26 on up 1 days 11:39:33
32 CPUs: 19.0% user, 0.0% nice, 12.8% sys, 67.4% spin, 0.0% intr, 0.8% idle
Memory: Real: 504M/7786M act/tot Free: 116G Cache: 6008M Swap: 0K/88G
cpu13 crashed because on mi_switch cpuswitch corrupted the registers.
CPU after 17 did not stop (probably because the cpu mondo broke down).
Three make processes are running:
pid 83460 cpu 13 is toast but somewhere between fork and exec.
pid 27987 cpu 4 is in uvmspace_exec()
pid 2638 cpu 9 is spinning on the kernel lock for some syscall
No process in ps /o seems to hold the KERNEL_LOCK which is a bit strange since
the parent make process is waiting for the lock.
--
:wq Claudio
ddb{13}> show panic
cpu2: uobjlk rwlock 0x4077ae3b6b0: enter write deadlock
*cpu13: mutex 0x0 not held in mi_switch
cpu18: Unable to broadcast mondo 1011cc4: 9
cpu20: kernel data fault: pc=1769a44 addr=1c78b7e000
cpu25: trap type 0x34 (mem address not aligned): pc=1063d70 npc=1063e0c pstate
=44820006<PRIV,IE>
cpu31: Unable to broadcast mondo 1011cc4: 9
ddb{13}> tr
sun4v_send_ipi(11, 1011ddc, 2710, 9, ffffffffffffffff, 0) at sun4v_send_ipi+0xa
c
db_enter_ddb(41b9fef8000, a, 1cbffc0, 1c80ac0, 1c189e0, 40000) at db_enter_ddb+
0x244
db_ktrap(101, 407659b1390, 1, 0, 407659b1610, 0) at db_ktrap+0x104
trap(407659b1390, 101, 1300ca4, 820006, 0, 1f) at trap+0x2d8
Lslowtrap_reenter(27, 41b9ff388ac, 17cf618, 407659b16d8, 1a06e90, 0) at Lslowtr
ap_reenter+0xf8
panic(17cf618, 0, 19f4810, 0, 100, 41b9ff388ac) at panic+0xcc
mi_switch(0, 0, 0, 0, 0, 0) at mi_switch+0x388
ddb{13}> ps /o
TID PID UID PRFLAGS PFLAGS CPU COMMAND
*203859 83460 21 0x100001 0 13 make
44218 27987 21 0x101005 0 4 make
490412 87307 21 0x3 0 5 cc
355576 75008 21 0x3 0 17 cc
355245 55821 21 0x3 0 20 cc
384888 87907 21 0x100003 0 18 cc
462853 25796 21 0x100003 0 6 ld
368951 38484 21 0x100003 0 11 cc1
419968 12491 21 0x100003 0 16 cc1
327300 77705 21 0x100003 0 3 cc1
422105 21158 21 0x100003 0 12 as
154033 17629 21 0x100003 0 7 cc1
26369 29404 21 0x100003 0 14 cc1
130757 4217 21 0x100003 0 10 cc1
257136 33437 21 0x100003 0 15 cc1
9396 70728 21 0x100003 0 1 cc1
313747 2638 21 0x10100003 0 9 make
510201 65806 115 0x100012 0 26 slaacd
171627 71222 115 0x100012 0 28 slaacd
39953 28730 0 0x14000 0x200 0 reaper
470293 89415 0 0x14000 0x200 23 softnet0
ddb{13}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
*83460 203859 2638 21 7 0x100001 make
27987 44218 2638 21 7 0x101005 make
87307 490412 2638 21 7 0x3 cc
29978 253097 2638 21 3 0x3 uobjlk cc
15695 197186 2638 21 3 0x3 uobjlk cc
75008 355576 2638 21 7 0x3 cc
55821 355245 2638 21 7 0x3 cc
87907 384888 2638 21 7 0x100003 cc
333 427413 90561 21 3 0x100083 piperd as
9370 367604 90561 21 3 0x100003 inode cc1
24712 51419 76483 21 3 0x100083 piperd as
74182 171943 76483 21 3 0x100003 inode cc1
25796 462853 2638 21 7 0x100003 ld
55683 169684 59514 21 3 0x100083 piperd as
64090 413637 59514 21 3 0x100003 inode cc1
90561 492487 2638 21 3 0x100083 wait cc
8353 382079 23335 21 3 0x100083 piperd as
76483 302250 2638 21 3 0x100083 wait cc
59514 166217 2638 21 3 0x100083 wait cc
38484 368951 23335 21 7 0x100003 cc1
23335 96000 2638 21 3 0x100083 wait cc
81439 105417 34605 21 3 0x100083 piperd as
6895 247934 54518 21 3 0x100083 piperd as
62855 97939 34605 21 3 0x100003 inode cc1
12491 419968 54518 21 7 0x100003 cc1
54518 166956 2638 21 3 0x100083 wait cc
34605 28207 2638 21 3 0x100083 wait cc
62623 342208 18323 21 3 0x100083 piperd as
77705 327300 18323 21 7 0x100003 cc1
18323 438745 2638 21 3 0x100083 wait cc
91545 37025 30776 21 3 0x100083 piperd as
84584 30157 30776 21 3 0x100003 inode cc1
30776 488756 2638 21 3 0x100083 wait cc
21158 422105 83687 21 7 0x100003 as
83687 510796 2638 21 3 0x100083 wait cc
37707 192885 77861 21 3 0x100083 piperd as
17629 154033 77861 21 7 0x100003 cc1
77861 96407 2638 21 3 0x100083 wait cc
45088 8998 2765 21 3 0x100083 piperd as
29404 26369 2765 21 7 0x100003 cc1
64386 326829 5796 21 3 0x100083 piperd as
4217 130757 5796 21 7 0x100003 cc1
2765 464726 2638 21 3 0x100083 wait cc
5796 19327 2638 21 3 0x100083 wait cc
98228 386403 63771 21 3 0x100083 piperd as
33437 257136 63771 21 7 0x100003 cc1
63771 91825 2638 21 3 0x100083 wait cc
80973 297569 12821 21 3 0x100083 piperd as
70728 9396 12821 21 7 0x100003 cc1
12821 365734 2638 21 3 0x100083 wait cc
35145 126344 2638 21 3 0x100083 wait cc
2638 313747 82086 21 7 0x10100003 make
82086 387179 7908 21 3 0x10008b sigsusp sh
7908 464733 51747 21 3 0x10008b sigsusp make
51747 461444 16433 0 3 0x10008b sigsusp sh
16433 350032 12765 0 3 0x10008b sigsusp make
12765 92966 66947 0 3 0x10008b sigsusp make
66947 354988 41804 0 3 0x100083 wait time
41804 291802 59417 0 3 0x100083 wait time
71036 327746 70600 0 3 0x100083 piperd tee
59417 497258 70600 0 3 0x100089 sigsusp sh
70600 511505 55184 0 3 0x10008b sigsusp sh
36395 193047 98843 1000 3 0x100083 kqread tail
98843 480682 85946 1000 3 0x10008b sigsusp ksh
60405 234733 99431 1000 3 0x100083 kqread top
99431 129956 85946 1000 3 0x10008b sigsusp ksh
85844 240848 6294 1000 3 0x83 kqread systat
6294 467906 85946 1000 3 0x10008b sigsusp ksh
55184 295301 85946 1000 3 0x10008b sigsusp ksh
85946 193395 1 1000 3 0x100080 kqread tmux
33451 84859 86833 1000 3 0x100083 kqread tmux
86833 124813 36245 1000 3 0x10008b sigsusp ksh
36245 293646 16037 1000 3 0x98 kqread sshd-session
16037 223564 8899 0 3 0x82 kqread sshd-session
42375 2491 1 0 3 0x100083 ttyin ksh
39678 457854 1 0 3 0x100098 kqread cron
26985 107453 1 99 3 0x1100090 kqread sndiod
91672 198225 1 110 3 0x100090 kqread sndiod
65750 404389 79014 95 3 0x1100092 kqread smtpd
34479 369135 79014 103 3 0x1100092 kqread smtpd
26626 31062 79014 95 3 0x1100092 kqread smtpd
6996 108112 79014 95 3 0x100092 kqread smtpd
91984 216217 79014 95 3 0x1100092 kqread smtpd
60436 393440 79014 95 3 0x1100092 kqread smtpd
79014 442861 1 0 3 0x100080 kqread smtpd
8899 161590 1 0 3 0x88 kqread sshd
15440 287074 1 0 3 0x100080 kqread ntpd
39263 361534 9686 83 3 0x100092 kqread ntpd
9686 409798 1 83 3 0x1100092 kqread ntpd
39393 284534 5827 74 3 0x1100092 bpf pflogd
5827 209189 1 0 3 0x80 sbwait pflogd
14302 155928 38669 73 3 0x1100090 kqread syslogd
38669 407555 1 0 3 0x100082 sbwait syslogd
44494 220072 1 0 3 0x100080 kqread resolvd
21339 505439 60763 77 3 0x100092 kqread dhcpleased
74356 361194 60763 77 3 0x100092 kqread dhcpleased
60763 325943 1 0 3 0x80 kqread dhcpleased
65806 510201 75067 115 7 0x100012 slaacd
71222 171627 75067 115 7 0x100012 slaacd
75067 124777 1 0 3 0x100080 sysctllk slaacd
80151 453697 0 0 3 0x14200 bored smr
24408 170266 0 0 3 0x14200 pgzero zerothread
36407 427572 0 0 3 0x14200 aiodoned aiodoned
62565 55260 0 0 3 0x14200 syncer update
64116 201669 0 0 3 0x14200 cleaner cleaner
28730 39953 0 0 7 0x14200 reaper
33431 157164 0 0 3 0x14200 pgdaemon pagedaemon
46161 472629 0 0 3 0x14200 usbtsk usbtask
2992 126411 0 0 3 0x14200 usbatsk usbatsk
84640 423001 0 0 3 0x14200 bored sensors
54658 520770 0 0 3 0x40014200 idle31
19399 499906 0 0 7 0x40014200 idle30
20858 482044 0 0 7 0x40014200 idle29
79829 50353 0 0 3 0x40014200 idle28
1546 195916 0 0 7 0x40014200 idle27
61902 85967 0 0 3 0x40014200 idle26
89974 484980 0 0 3 0x40014200 idle25
2447 481193 0 0 7 0x40014200 idle24
28705 54131 0 0 3 0x40014200 idle23
14687 482436 0 0 7 0x40014200 idle22
80080 523569 0 0 7 0x40014200 idle21
73543 477047 0 0 3 0x40014200 idle20
90851 461175 0 0 7 0x40014200 idle19
67402 467610 0 0 3 0x40014200 idle18
92980 470105 0 0 3 0x40014200 idle17
25688 373 0 0 3 0x40014200 idle16
2140 359395 0 0 3 0x40014200 idle15
30307 298916 0 0 3 0x40014200 idle14
78276 222991 0 0 3 0x40014200 idle13
53698 232569 0 0 3 0x40014200 idle12
25040 169137 0 0 3 0x40014200 idle11
52331 91953 0 0 3 0x40014200 idle10
80801 298113 0 0 3 0x40014200 idle9
71270 356160 0 0 3 0x40014200 idle8
37142 63435 0 0 3 0x40014200 idle7
41282 158113 0 0 3 0x40014200 idle6
46031 294695 0 0 3 0x40014200 idle5
68521 268223 0 0 3 0x40014200 idle4
61895 101641 0 0 3 0x40014200 idle3
80865 338317 0 0 3 0x40014200 idle2
33775 6315 0 0 3 0x40014200 idle1
3052 79059 0 0 3 0x14200 bored softnet7
27723 260168 0 0 3 0x14200 bored softnet6
40405 54896 0 0 3 0x14200 bored softnet5
46409 135364 0 0 3 0x14200 bored softnet4
54015 335233 0 0 3 0x14200 bored softnet3
75531 132971 0 0 3 0x14200 bored softnet2
55071 464772 0 0 3 0x14200 bored softnet1
89415 470293 0 0 7 0x14200 softnet0
59654 170171 0 0 3 0x14200 bored systqmp
2035 521522 0 0 3 0x14200 bored systq
44922 452892 0 0 3 0x14200 tmoslp softclockmp
13424 132664 0 0 3 0x40014200 tmoslp softclock
92466 443387 0 0 3 0x40014200 idle0
15310 87170 0 0 3 0x14200 kmalloc kmthread
1 308242 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{13}> show uvm
Current UVM status:
pagesize=8192 (0x2000), pagemask=0x1fff, pageshift=13
16258622 VM pages: 33991 active, 41539 inactive, 1 wired, 15294921 free (1912
382 zero)
freemin=541954, free-target=722605, inactive-target=0, wired-max=5419540
faults=-543425818, traps=1357591386, intrs=972970408, ctxswitch=642541198 fpu
switch=3254093
softint=32992076, syscalls=-1651139276, kmapent=20
fault counts:
noram=0, noanon=0, noamap=0, pgwait=0, pgrele=0
relocks=6641422(72444), upgrades=0(0) anget(retries)=812094109(0), amapcopy
=304977889
neighbor anon/obj pg=151753437/735668315, gets(lock/unlock)=493797378/67142
16
cases: anon=730615483, anoncow=81478626, obj=454908346, prcopy=38816238, pr
zero=-1849244513
daemon and swap counts:
woke=0, revs=0, scans=0, obscans=0, anscans=0
busy=0, freed=0, reactivate=0, deactivate=0
pageouts=0, pending=0, nswget=0
nswapdev=1
swpages=11590896, swpginuse=0, swpgonly=0 paging=0
kernel pointers:
objs(kern)=0x1c72d40
ddb{13}> show pool
POOL panic: kernel data fault: pc=1369664 addr=300ffff83286000
Stopped at db_enter+0x8: nop
data_access_fault(407659b0340, 31, 1369664, 300ffff83286000, 300ffff83286000, 1
) at data_access_fault+0x2f0
sun4v_datatrap(300ffff8328700f, 300ffff83287010, 1, 0, 407659b06f0, 0) at sun4v
_datatrap+0x200
kprintf(5, 14, 0, 0, 0, 0) at kprintf+0xf94
db_printf(18322f8, 300ffff8328700f, 86290001, 81cfe008, 407659b0920, 0) at db_p
rintf+0x40
pool_print1(1300ca8, 407659b0800, 1263720, 407659b0800, 0, 1c866d8) at pool_pri
nt1+0x54
db_command(1c24a88, 0, 0, 0, 1300ca8, 407659b0958) at db_command+0x134
db_command_loop(91, 407659b1650, e, 5, 1263720, 1937900) at db_command_loop+0x1
0c
db_trap(1c26000, 0, 0, 0, 0, 1f) at db_trap+0x214
db_ktrap(101, 407659b0cc0, 1, 407659b16d8, 407659b0f38, 0) at db_ktrap+0x1b4
trap(407659b0cc0, 101, 1300ca4, 820006, 0, 0) at trap+0x2d8
Lslowtrap_reenter(1, 14, 0, 0, 0, 1) at Lslowtrap_reenter+0xf8
panic(18374e0, 1011ddc, 11, 9, 104, 41b9ff388ac) at panic+0xcc
sun4v_send_ipi(11, 1011ddc, 2710, 9, ffffffffffffffff, 0) at sun4v_send_ipi+0xa
c
db_enter_ddb(41b9fef8000, a, 1cbffc0, 1c80ac0, 1c189e0, 40000) at db_enter_ddb+
0x244
ddb{13}> x /i 0x1369664
strlen+0x4: ldsb [%o0 + %g0], %o2
ddb{13}> show panic
cpu2: uobjlk rwlock 0x4077ae3b6b0: enter write deadlock
*cpu13: mutex 0x0 not held in mi_switch
cpu18: Unable to broadcast mondo 1011cc4: 9
cpu20: kernel data fault: pc=1769a44 addr=1c78b7e000
cpu25: trap type 0x34 (mem address not aligned): pc=1063d70 npc=1063e0c pstate
=44820006<PRIV,IE>
cpu31: Unable to broadcast mondo 1011cc4: 9
ddb{13}> x /i 0x1769a44
mi_switch+0x164: ld [%g1 + 0xd80], %g2
ddb{13}> x /i 1063d70
sched_steal_proc+0xd0: ld [%l0 + 0x60], %g1
ddb{13}> show proc
PROC (make) tid=203859 pid=83460 tcnt=1 stat=onproc
flags process=100001<CONTROLT,PLEDGE> proc=0
runpri=86, usrpri=86, slppri=32, nice=20
wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0
forw=0xffffffffffffffff, list=0x4077ca59360,0x1c7e858
process=0x407714e8920 user=0x407659ae000, vmspace=0x40774792310
estcpu=36, cpticks=0, pctcpu=0.0, user=0, sys=0, intr=0
ddb{13}> mach ddbcpu 0
Stopped at __mp_lock+0x64: nop
sparc_intr_retry(1c19850, 0, 1813488, 40770b4c000, 1be, 1cd2f90) at sparc_intr_
retry+0x58
msleep_nsec(1c19860, 1c19850, 4, 1813488, 0, 6) at msleep_nsec+0x88
reaper(4008f8842f0, 4008f8842f0, 381, 100, 1c19860, 4000) at reaper+0xa0
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x10
ddb{0}> mach ddbcpu 1
Stopped at hv_cpu_mondo_send+0xc: nop
pmap_enter(40778204240, 294b78000, 7e16b5a44000, 407713f5a00, 21, 4) at pmap_en
ter+0x45c
uvm_fault_lower(407713f5d08, 407713f5cd0, 4076cb4f440, 407713f5bd0, 407713f5af0
, 1cb9440) at uvm_fault_lower+0x4a4
uvm_fault(0, 407713f5d08, 0, 0, 11782e0, 2) at uvm_fault+0x1b8
data_access_fault(407713f5ee0, 31, 49f8e0, 294b78dc9, 294b78000, 1) at data_acc
ess_fault+0x184
sun4v_datatrap(294b78000, ffffffff, 206260, 2042d8cd0, 100, 268) at sun4v_datat
rap+0x200
ddb{1}> mach ddbcpu 2
Stopped at hv_cons_putchar+0xc: nop
cnputc(74, 74, 0, 40787973b30, 11782e0, 3) at cnputc+0x30
db_putchar(74, 14, 1, 0, 40762b1b820, 0) at db_putchar+0x140
kprintf(27, 14, 0, 0, 0, 31) at kprintf+0x99c
db_printf(18c0208, 41b9ffe88ac, 19031a8, 40762b1b8e8, 0, 41b9ff388ac) at db_pri
ntf+0x40
panic(19031a8, 1915230, 4077ae3b6b0, 0, 104, 41b9ffe88ac) at panic+0xa8
rw_do_enter_write(4077ae3b6b0, 1, 40762b1bd30, 0, 0, 0) at rw_do_enter_write+0x
1e4
uvm_fault_lower_lookup(40762b1bd08, 40762b1bcd0, 40762b1bbd0, 0, 0, 1a06e90) at
uvm_fault_lower_lookup+0x2c
uvm_fault_lower(40762b1bd08, 40762b1bcd0, 0, 40762b1bbd0, 40762b1baf0, 1cb9440)
at uvm_fault_lower+0x44
uvm_fault(0, 40762b1bd08, 0, 40787973b30, 11782e0, 3) at uvm_fault+0x1b8
data_access_fault(40762b1bee0, 31, 22300b508, 221afae65, 221afa000, 1) at data_
access_fault+0x184
sun4v_datatrap(4a31, 2e, 1, a3, 2e29b2, 0) at sun4v_datatrap+0x200
ddb{2}> mach ddbcpu 4
Stopped at delay+0x40: rdpr %tick, %o2
pmap_remove(6a5220c000, 6a52200000, 6a5221e000, 7fffffffe000, 19470c8, 1cbf000)
at pmap_remove+0x1e0
uvm_unmap_kill_entry_withlock(40774793a10, 40786305010, 1, 4076265d728, 0, 1) a
t uvm_unmap_kill_entry_withlock+0xd0
uvm_unmap_remove(40774793a10, ffffffffffffe000, 40786305010, 4076265d728, 1, 0)
at uvm_unmap_remove+0x244
uvmspace_exec(4077ca59360, 2000, ffffffffffffe000, 24f0, 0, 6d4e144948) at uvms
pace_exec+0x154
sys_execve(c, 6d4e144940, 4008f8d6930, 4011cff6000, 3c, 40770f21b10) at sys_exe
cve+0x5ac
syscall(4076265dee0, 1a06e90, 4076265df40, 1c00, 40770f21f10, 17d06e0) at sysca
ll+0x3c0
syscall_setup(fffffea81d8d35e8, 6cea8ae000, 6d4e1448c0, 1002, ffffffffffffffff,
0) at syscall_setup+0x124
ddb{4}> show proc
PROC (make) tid=44218 pid=27987 tcnt=1 stat=onproc
flags process=101005<CONTROLT,INEXEC,SINGLEEXIT,PLEDGE> proc=0
runpri=36, usrpri=86, slppri=36, nice=20
wchan=0x0, wmesg=, ps_single=0x4077ca59360 scnt=0 ecnt=0
forw=0xffffffffffffffff, list=0x4077ca58090,0x4077ca58b60
process=0x40770f21b10 user=0x4076265a000, vmspace=0x40774793a10
estcpu=36, cpticks=5159913, pctcpu=0.0, user=0, sys=5159913, intr=0
ddb{4}> mach ddbcpu 9
Stopped at __mp_lock+0x64: nop
syscall_setup(ffffffffffffffff, fffffea81d8d3dd4, 1, 0, 17a8, 6a52530248) at sy
scall_setup+0x124
ddb{9}> show proc
PROC (make) tid=313747 pid=2638 tcnt=1 stat=onproc
flags process=10100003<CONTROLT,EXEC,PLEDGE,WAITEVENT> proc=0
runpri=32, usrpri=86, slppri=32, nice=20
wchan=0x0, wmesg=, ps_single=0x0 scnt=0 ecnt=0
forw=0xffffffffffffffff, list=0x4076ab0ee10,0x40774512890
process=0x4076ab428f0 user=0x4076093a000, vmspace=0x4008f606ff0
estcpu=36, cpticks=5163642, pctcpu=9.6, user=30, sys=5163608, intr=3
