On May 13, 2023, at 9:16 AM, Kurt Miller <k...@intricatesoftware.com> wrote:
>
> On May 12, 2023, at 10:26 AM, Martin Pieuchot <m...@openbsd.org> wrote:
>>
>> On 09/05/23(Tue) 20:02, Kurt Miller wrote:
>>> While building devel/jdk/1.8 on May 3rd snapshot I noticed the build
>>> freezing
>>> and processes getting stuck like ps. After enabling ddb.console I was able
>>> to
>>> reproduce the livelock and capture cpu traces. Dmesg at the end.
>>> Let me know if more information is needed as this appears to be rather
>>> reproducible on my T4-1.
>>
>> It seems that all CPUs are waiting for the KERNEL_LOCK(). Doing ps /o
>> in ddb(4) should show us which CPU is currently holding it. I can't
>> figure it out just by looking at the traces below.
>>
Here’s another one. Unfortunately, I got a ddb panic while switching cpus again.
This time I did the ps /o first. I can keep trying for a clean set of traces
without
ddb panic’s if that is helpful.
login: Stopped at db_enter+0x8: nop
ddb{0}> ps /o
TID PID UID PRFLAGS PFLAGS CPU COMMAND
284507 78236 55 0x200003 0x4000000 26 java
278836 78236 55 0x200003 0x4000000 32 java
481738 78236 55 0x200003 0x4000000 30 java
448486 78236 55 0x200003 0x4000000 25 java
31673 78236 55 0x200003 0x4000000 15 java
354968 78236 55 0x200003 0x4000000 9 java
244070 78236 55 0x200003 0x4000000 52 java
27520 78236 55 0x200003 0x4000000 5 java
160016 78236 55 0x200003 0x4000000 31 java
176767 78236 55 0x200003 0x4000000 16 java
107452 78236 55 0x200003 0x4000000 24 java
223 78236 55 0x200003 0x4000000 6 java
57612 78236 55 0x200003 0x4000000 8 java
450768 78236 55 0x200003 0x4000000 35 java
283448 78236 55 0x200003 0x4000000 40 java
286351 78236 55 0x200003 0x4000000 39 java
1388 78236 55 0x200003 0x4000000 1 java
112191 78236 55 0x200003 0x4000000 13 java
ddb{0}> tr
vcons_softintr(400151b8a00, 1c91a58, 20, 400e2324ff0, 1146040, 6) at
vcons_softintr+0x3c
intr_handler(2017ec8, 400151afa00, 1edca0da, 1c8ddf0, 0, ac) at
intr_handler+0x50
sparc_intr_retry(0, 0, 1edc7f69, 893a1a216c, 1c00, 12) at sparc_intr_retry+0x5c
cpu_idle_cycle(1c91a38, 2018000, 17fe630, 1c8ddf0, 0, 19a1530) at
cpu_idle_cycle+0x2c
sched_idle(2018360, 4001516ab00, 17e7bd8, 0, 1c76566, 3b9ac800) at
sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{0}> machine ddbcpu 0x1
Stopped at __mp_lock+0x68: ld [%o0 + 0x800], %g1
ddb{1}> bt
rw_enter(400e239b8e0, 19a1530, 1cd1000, 0, 1c0c000, 0) at rw_enter+0x1fc
uvm_fault_lower_lookup(400e247dcd0, 400e247dd08, 400e247dbd0, 0, 0, 4) at
uvm_fault_lower_lookup+0x2c
uvm_fault_lower(400e2324ff0, 400e247dd08, 400e247dbd0, 0, 1c02000, 1c80230) at
uvm_fault_lower+0x3c
uvm_fault(0, 400e247dcd0, 0, 400e2324ff0, 1146040, 1) at uvm_fault+0x1bc
text_access_fault(400e247ded0, 9, 88ce3e62ac, 0, 0, 0) at
text_access_fault+0x114
sun4v_texttrap(898179b060, 52a, 893a1a2168, 893a1a216c, 0, 2b) at
sun4v_texttrap+0x1fc
ddb{1}> machine ddbcpu 0x2
Sptanoippced: att ra p t yp e 0x 14 7 ( *t r a p ) : p c = 1 6
6 8 a e c__ m p_ lo ck +0 x 68 : l d [ %o 0 +
0 x8 00 ], %g 1
TI D P IDn p c = 1 6U6ID8 a f 0 p sPRtFLaAGtSe = 8 2
0 0 0PF6LA<GSP R I VCP,U I EC>OM
ND
MA
284507 78236 55 0x200003 0x4000000 26 java
278836 78236 55 0x200003 0x4000000 32 java
481738 78236 55 0x200003 0x4000000 30 java
448486 78236 55 0x200003 0x4000000 25 java
31673 78236 55 0x200003 0x4000000 15 java
354968 78236 55 0x200003 0x4000000 9 java
244070 78236 55 0x200003 0x4000000 52 java
27520 78236 55 0x200003 0x4000000 5 java
160016 78236 55 0x200003 0x4000000 31 java
176767 78236 55 0x200003 0x4000000 16 java
107452 78236 55 0x200003 0x4000000 24 java
223 78236 55 0x200003 0x4000000 6 java
57612 78236 55 0x200003 0x4000000 8 java
450768 78236 55 0x200003 0x4000000 35 java
283448 78236 55 0x200003 0x4000000 40 java
286351 78236 55 0x200003 0x4000000 39 java
1388 78236 55 0x200003 0x4000000 1 java
112191 78236 55 0x200003 0x4000000 13 java
mi_switch(1c88c98, 400e0728000, 17fe630, 1c8ddf0, 400e0db5204, 19a1530) at
mi_switch+0x2ac
sched_idle(400e0728360, 4001515db90, 17e7bd8, 400151269f0, 0, 3b9ac800) at
sched_idle+0x250
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
https://www.openbsd.org/ddb.html describes the minimum info required in bug
reports. Insufficient info makes it difficult to find and fix bugs.
ddb{2}> bt
mi_switch(1c88c98, 400e0728000, 17fe630, 1c8ddf0, 400e0db5204, 19a1530) at
mi_switch+0x2ac
sched_idle(400e0728360, 4001515db90, 17e7bd8, 400151269f0, 0, 3b9ac800) at
sched_idle+0x250
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{2}> machine ddbcpu 0x3
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0738360, 4001515d610, 17e7bd8, 1c00, 18f4788, 3b9ac800) at
sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{3}> bt
sched_idle(400e0738360, 4001515d610, 17e7bd8, 1c00, 18f4788, 3b9ac800) at
sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{3}> machine ddbcpu 0x4
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0748360, 4001515d090, 17e7bd8, 1c00, 18f4788, 3b9ac800) at
sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{4}> machine ddbcpu 0x5
Stopped at __mp_lock+0x68: ld [%o0 + 0x800], %g1
rw_enter(400e239b8e0, 19a1530, 1cd1000, 0, 1c0c000, 0) at rw_enter+0x1fc
uvm_fault_lower_lookup(400e0fc1cd0, 400e0fc1d08, 400e0fc1bd0, 0, 0, 4) at
uvm_fault_lower_lookup+0x2c
uvm_fault_lower(400e2324ff0, 400e0fc1d08, 400e0fc1bd0, 0, 1c02000, 1c80230) at
uvm_fault_lower+0x3c
uvm_fault(0, 400e0fc1cd0, 0, 400e2324ff0, 1146040, 1) at uvm_fault+0x1bc
text_access_fault(400e0fc1ed0, 9, 88ce3e62ac, 0, 0, 0) at
text_access_fault+0x114
sun4v_texttrap(898179b060, 52a, 893a1a2168, 893a1a216c, 0, 2b) at
sun4v_texttrap+0x1fc
ddb{5}> machine ddbcpu 0x6
Stopped at db_enter+0x8: nop
trap(400e0fed9b0, 147, 1668aec, 820006, 0, 0) at trap+0x328
Lslowtrap_reenter(1cd1b18, 400e1fc0114, 400e0fedcf0, 400e2324ff0, 1146040, 6)
at Lslowtrap_reenter+0xf8
trapsignal(400e204d1b0, 5, 0, 1, 88cea9ecdc, 0) at trapsignal+0xe8
trap(400e0feded0, 101, 88cea9ecdc, 99000002, 0, 2b) at trap+0x500
Lslowtrap_reenter(0, ffffffffffffffec, 11e5070ee474, 8, 890378e7b4, 88a443e048)
at Lslowtrap_reenter+0xf8
ddb{6}> machine ddbcpu 0x7
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0778360, 4001515c590, 17e7bd8, 1c00, 18f4788, 3b9ac800) at
sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{7}> machine ddbcpu 0x8
Stopped at __mp_lock+0x64: nop
mi_switch(400e0788000, 400e204c6b0, 60, 1146080, 1146040, 19a1530) at
mi_switch+0x2ac
sys_sched_yield(400e204c6b0, 400e0ff1db0, 400e0ff1df0, 199daf8, 1146080, ccf)
at sys_sched_yield+0x70
syscall(400e0ff1ed0, 52a, 893a1a2168, 893a1a216c, 0, 19a1530) at syscall+0x464
syscall_setup(88fe6f65c0, fffffffffffffff6, 11e4faf167ad, 0, 893b6ce284,
88fe22f708) at syscall_setup+0x134
ddb{8}> machine ddbcpu 0x9
Stopped at __mp_lock+0x64: nop
rw_enter(400e1ff1c70, 19a1530, 1cd1000, 20, 1c0c000, 0) at rw_enter+0x1fc
vm_map_lock_ln(400e1ff1b90, 18f7a38, 6e7, 0, 0, 0) at vm_map_lock_ln+0x3c
uvmfault_lookup(400e37e5cd0, 1, 6ae, 0, 0, 0) at uvmfault_lookup+0x50
uvm_fault_check(e, 400e37e5d08, 400e37e5d30, 1146040, 21, 3) at
uvm_fault_check+0x3b4
uvm_fault(400e1ff1b90, 400e37e5cd0, 0, 1, 1146040, 1) at uvm_fault+0xe0
data_access_fault(400e37e5ed0, 31, 88ce96d8c4, 895f4402ac, 895f440000, 1) at
data_access_fault+0x184
sun4v_datatrap(400e37e5ed0, 31, 88ce924a5c, 895db402ac, 895db40000, 1) at
sun4v_datatrap+0x210
ddb{9}> machine ddbcpu 0xa
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e07a8360, 4001515cdd0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{10}> machine ddbcpu 0xb
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e07b8360, 4001515d350, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{11}> machine ddbcpu 0xc
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e07c8360, 4001515d8d0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{12}> machine ddbcpu 0xd
Stopped at __mp_lock+0x68: ld [%o0 + 0x800], %g1
uvm_fault_lower(0, 400e2495d08, 40010e6a170, 40010e6a1b8, 1c02000, 1c80230) at
uvm_fault_lower+0x880
uvm_fault(0, 400e2495cd0, 0, 400e2324ff0, 1146040, 1) at uvm_fault+0x1bc
text_access_fault(400e2495ed0, 9, 88ce3e62ac, 0, 0, 0) at
text_access_fault+0x114
sun4v_texttrap(898179b060, 52a, 893a1a2168, 893a1a216c, 0, 2b) at
sun4v_texttrap+0x1fc
ddb{13}> machine ddbcpu 0xe
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e07e8360, 40015156b20, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{14}> machine ddbcpu 0xf
Stopped at __mp_lock+0x68: ld [%o0 + 0x800], %g1
rw_enter(400e239b8e0, 19a1530, 1cd1000, 0, 1c0c000, 0) at rw_enter+0x1fc
uvm_fault_lower_lookup(400e37ddcd0, 400e37ddd08, 400e37ddbd0, 0, 0, 4) at
uvm_fault_lower_lookup+0x2c
uvm_fault_lower(400e2324ff0, 400e37ddd08, 400e37ddbd0, 0, 1c02000, 1c80230) at
uvm_fault_lower+0x3c
uvm_fault(0, 400e37ddcd0, 0, 400e2324ff0, 1146040, 1) at uvm_fault+0x1bc
text_access_fault(400e37dded0, 9, 88ce3e62ac, 0, 0, 0) at
text_access_fault+0x114
sun4v_texttrap(898179b060, 52a, 893a1a2168, 893a1a216c, 0, 19a1530) at
sun4v_texttrap+0x1fc
ddb{15}> machine ddbcpu 0x10
Stopped at __mp_lock+0x68: ld [%o0 + 0x800], %g1
rw_enter(400e1ff1c70, 19a1530, 1cd1000, 20, 1c0c000, 0) at rw_enter+0x1fc
vm_map_lock_ln(400e1ff1b90, 18f7a38, 6e7, 0, 1c0c000, 0) at vm_map_lock_ln+0x3c
uvmfault_lookup(400e0fc9cd0, 1, 6ae, 23a78, 23970, 88cf13c078) at
uvmfault_lookup+0x50
uvm_fault_check(e, 400e0fc9d08, 400e0fc9d30, 1146040, 21, 1c00) at
uvm_fault_check+0x3b4
uvm_fault(400e1ff1b90, 400e0fc9cd0, 0, 1, 1146040, 1) at uvm_fault+0xe0
data_access_fault(400e0fc9ed0, 31, 88ce96d8c4, 895f3e82ac, 895f3e8000, 1) at
data_access_fault+0x184
sun4v_datatrap(400e0fc9ed0, 31, d7, d7, 895f3d2000, 1) at sun4v_datatrap+0x210
ddb{16}> machine ddbcpu 0x11
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0818360, 40015156020, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{17}> machine ddbcpu 0x12
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0828360, 400151562e0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{18}> machine ddbcpu 0x13
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0838360, 40015156de0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{19}> machine ddbcpu 0x14
Stopped at __mp_lock+0x64: nop
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{20}> machine ddbcpu 0x15
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0858360, 40015157360, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{21}> machine ddbcpu 0x16
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0868360, 40015157620, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{22}> machine ddbcpu 0x17
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0878360, 40015157ba0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{23}> machine ddbcpu 0x18
Stopped at __mp_lock+0x64: nop
rw_enter(400e239b8e0, 19a1530, 1cd1000, 0, 1c0c000, 0) at rw_enter+0x1fc
uvm_fault_lower_lookup(400e3581cd0, 400e3581d08, 400e3581bd0, 0, 0, 4) at
uvm_fault_lower_lookup+0x2c
uvm_fault_lower(400e2324ff0, 400e3581d08, 400e3581bd0, 0, 1c02000, 1c80230) at
uvm_fault_lower+0x3c
uvm_fault(0, 400e3581cd0, 0, 400e2324ff0, 1146040, 1) at uvm_fault+0x1bc
text_access_fault(400e3581ed0, 9, 88ce3e62ac, 0, 0, 0) at
text_access_fault+0x114
sun4v_texttrap(898179b060, 52a, 893a1a2168, 893a1a216c, 0, 2b) at
sun4v_texttrap+0x1fc
ddb{24}> machine ddbcpu 0x19
Stopped at __mp_lock+0x64: nop
rw_enter(400e239b8e0, 19a1530, 1cd1000, 0, 1c0c000, 0) at rw_enter+0x1fc
uvm_fault_lower_lookup(400e3571cd0, 400e3571d08, 400e3571bd0, 0, 0, 4) at
uvm_fault_lower_lookup+0x2c
uvm_fault_lower(400e2324ff0, 400e3571d08, 400e3571bd0, 0, 1c02000, 1c80230) at
uvm_fault_lower+0x3c
uvm_fault(0, 400e3571cd0, 0, 400e2324ff0, 1146040, 1) at uvm_fault+0x1bc
text_access_fault(400e3571ed0, 9, 88ce3e62ac, 0, 0, 0) at
text_access_fault+0x114
sun4v_texttrap(898179b060, 52a, 893a1a2168, 893a1a216c, 0, 19a1530) at
sun4v_texttrap+0x1fc
ddb{25}> machine ddbcpu 0x1a
Stopped at __mp_lock+0x70: bne,pt __mp_lock+0x5c
clockintr_statclock(1, 400e08a7ec8, 17eea88, 3d, 10ae, 42) at
clockintr_statclock+0x48
clockintr_dispatch(400e08a7ec8, 1, 9, 3f2998000, ffffffffffffffff, 1d71f30) at
clockintr_dispatch+0x1f8
sys_tickintr(1, 88ce3e6000, 3f2998000, 400e3301a00, 24, 4) at sys_tickintr+0x8
intr_handler(400e08a7ec8, 400e08a88d8, 1edca10e, 0, 1c02000, 1c80230) at
intr_handler+0x50
sparc_intr_retry(1c0d8f8, 400e3301cd0, 1c0d8f8, 400e2324ff0, 1146040, 1) at
sparc_intr_retry+0x5c
db_enter_ddb(1ca7000, e, 40015162080, 400e1ff1b90, 0, 6) at db_enter_ddb+0xbc
db_ktrap(101, 400e3301750, 1932c70, 3, 1f94cc, 1cb9040) at db_ktrap+0x104
trap(400e3301750, 101, 1013da4, 44820006, 1f94cc, 1cb9040) at trap+0x2cc
Lslowtrap_reenter(10ae, a181aed34231, a181aed3426f, 3d, 10ae, 42) at
Lslowtrap_reenter+0xf8
sun4v_broadcast_ipi(1011eac, 1, 9, 3f2998000, ffffffffffffffff, 1d71f30) at
sun4v_broadcast_ipi+0xac
pmap_enter(40015162080, 88ce3e6000, 3f2998000, 400e3301a00, 24, 4) at
pmap_enter+0x45c
uvm_fault_lower(400e2324ff0, 400e3301d08, 40010e6a170, 0, 1c02000, 1c80230) at
uvm_fault_lower+0x198
uvm_fault(0, 400e3301cd0, 0, 400e2324ff0, 1146040, 1) at uvm_fault+0x1bc
text_access_fault(400e3301ed0, 9, 88ce3e62ac, 0, 0, 0) at
text_access_fault+0x114
ddb{26}> machine ddbcpu 0x1b
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e08b8360, 400151510b0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{27}> machine ddbcpu 0x1c
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e08c8360, 40015150b30, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{28}> machine ddbcpu 0x1d
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e08d8360, 40015150870, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{29}> machine ddbcpu 0x1e
Stopped at __mp_lock+0x64: nop
rw_enter(400e239b8e0, 19a1530, 1cd1000, 0, 1c0c000, 0) at rw_enter+0x1fc
uvm_fault_lower_lookup(400e2d0dcd0, 400e2d0dd08, 400e2d0dbd0, 0, 0, 4) at
uvm_fault_lower_lookup+0x2c
uvm_fault_lower(400e2324ff0, 400e2d0dd08, 400e2d0dbd0, 0, 1c02000, 1c80230) at
uvm_fault_lower+0x3c
uvm_fault(0, 400e2d0dcd0, 0, 400e2324ff0, 1146040, 1) at uvm_fault+0x1bc
text_access_fault(400e2d0ded0, 9, 88ce3e62ac, 0, 0, 0) at
text_access_fault+0x114
sun4v_texttrap(898179b060, 52a, 893a1a2168, 893a1a216c, 0, 2b) at
sun4v_texttrap+0x1fc
ddb{30}> machine ddbcpu 0x1f
Stopped at __mp_lock+0x68: ld [%o0 + 0x800], %g1
rw_enter(400e239b8e0, 19a1530, 1cd1000, 0, 1c0c000, 0) at rw_enter+0x1fc
uvm_fault_lower_lookup(400e0fc5cd0, 400e0fc5d08, 400e0fc5bd0, 0, 0, 4) at
uvm_fault_lower_lookup+0x2c
uvm_fault_lower(400e2324ff0, 400e0fc5d08, 400e0fc5bd0, 0, 1c02000, 1c80230) at
uvm_fault_lower+0x3c
uvm_fault(0, 400e0fc5cd0, 0, 400e2324ff0, 1146040, 1) at uvm_fault+0x1bc
text_access_fault(400e0fc5ed0, 9, 88ce3e62ac, 0, 0, 0) at
text_access_fault+0x114
sun4v_texttrap(898179b060, 52a, 893a1a2168, 893a1a216c, 0, 19a1530) at
sun4v_texttrap+0x1fc
ddb{31}> machine ddbcpu 0x20
Stopped at __mp_lock+0x6c: subcc %g3, %g1, %g0
trap(400e24b9ed0, 101, 88ce22c660, 99000002, 0, 2b) at trap+0x500
Lslowtrap_reenter(0, fffffffffffffff5, 11e506426a5a, 13, 8940957104,
88fe22f548) at Lslowtrap_reenter+0xf8
ddb{32}> machine ddbcpu 0x20
Invalid cpu 32
ddb{32}> machine ddbcpu 0x21
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0918360, 40015150df0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{33}> machine ddbcpu 0x22
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0928360, 400151518f0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{34}> machine ddbcpu 0x23
Stopped at __mp_lock+0x64: nop
syscall(400e0ffded0, 52a, 893a1a2168, 893a1a216c, 0, 19a1530) at syscall+0x464
syscall_setup(88fe6f65c0, ffffffffffffffef, 11e4faf13a68, c, 89a4b4ee14,
88a443e048) at syscall_setup+0x134
ddb{35}> machine ddbcpu 0x24
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0948360, 4001514b0c0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{36}> machine ddbcpu 0x25
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0958360, 4001514ab40, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{37}> machine ddbcpu 0x26
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0968360, 4001514a300, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{38}> machine ddbcpu 0x27
Stopped at __mp_lock+0x64: nop
rw_enter(400e239b8e0, 19a1530, 1cd1000, 0, 1c0c000, 0) at rw_enter+0x1fc
uvm_fault_lower_lookup(400e2475cd0, 400e2475d08, 400e2475bd0, 0, 0, 4) at
uvm_fault_lower_lookup+0x2c
uvm_fault_lower(400e2324ff0, 400e2475d08, 400e2475bd0, 0, 1c02000, 1c80230) at
uvm_fault_lower+0x3c
uvm_fault(0, 400e2475cd0, 0, 400e2324ff0, 1146040, 1) at uvm_fault+0x1bc
text_access_fault(400e2475ed0, 9, 88ce3e62ac, 0, 0, 0) at
text_access_fault+0x114
sun4v_texttrap(898179b060, 52a, 893a1a2168, 893a1a216c, 0, 2b) at
sun4v_texttrap+0x1fc
ddb{39}> machine ddbcpu 0x28
Stopped at __mp_lock+0x5c: wr %g0, 0x80, %
rw_enter(400e239b8e0, 19a1530, 1cd1000, 0, 1c0c000, 0) at rw_enter+0x1fc
uvm_fault_lower_lookup(400e246dcd0, 400e246dd08, 400e246dbd0, 0, 0, 4) at
uvm_fault_lower_lookup+0x2c
uvm_fault_lower(400e2324ff0, 400e246dd08, 400e246dbd0, 0, 1c02000, 1c80230) at
uvm_fault_lower+0x3c
uvm_fault(0, 400e246dcd0, 0, 400e2324ff0, 1146040, 1) at uvm_fault+0x1bc
text_access_fault(400e246ded0, 9, 88ce3e62ac, 0, 0, 0) at
text_access_fault+0x114
sun4v_texttrap(898179b060, 52a, 893a1a2168, 893a1a216c, 0, 19a1530) at
sun4v_texttrap+0x1fc
ddb{40}> machine ddbcpu 0x29
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0998360, 4001514a880, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{41}> machine ddbcpu 0x2a
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e09a8360, 4001514ae00, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{42}> machine ddbcpu 0x2b
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e09b8360, 4001514b380, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{43}> machine ddbcpu 0x2c
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e09c8360, 4001514b640, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{44}> machine ddbcpu 0x2d
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e09d8360, 4001514b900, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{45}> machine ddbcpu 0x2e
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e09e8360, 40015145bd0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{46}> machine ddbcpu 0x2f
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e09f8360, 40015145910, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{47}> machine ddbcpu 0x30
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a08360, 400151450d0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{48}> machine ddbcpu 0x31
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a18360, 40015144e10, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{49}> machine ddbcpu 0x32
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a28360, 40015144890, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{50}> machine ddbcpu 0x33
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a38360, 40015144050, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{51}> machine ddbcpu 0x34
Stopped at __mp_lock+0x70: bne,pt __mp_lock+0x5c
rw_enter(400e239b8e0, 19a1530, 1cd1000, 0, 1c0c000, 0) at rw_enter+0x1fc
uvm_fault_lower_lookup(400e394dcd0, 400e394dd08, 400e394dbd0, 0, 0, 4) at
uvm_fault_lower_lookup+0x2c
uvm_fault_lower(400e2324ff0, 400e394dd08, 400e394dbd0, 0, 1c02000, 1c80230) at
uvm_fault_lower+0x3c
uvm_fault(0, 400e394dcd0, 0, 400e2324ff0, 1146040, 1) at uvm_fault+0x1bc
text_access_fault(400e394ded0, 9, 88ce3e62ac, 0, 0, 0) at
text_access_fault+0x114
sun4v_texttrap(898179b060, 52a, 893a1a2168, 893a1a216c, 0, 19a1530) at
sun4v_texttrap+0x1fc
ddb{52}> machine ddbcpu 0x35
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a58360, 400151445d0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{53}> machine ddbcpu 0x36
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a68360, 40015144b50, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{54}> machine ddbcpu 0x37
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a78360, 40015145390, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{55}> machine ddbcpu 0x38
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a88360, 40015145650, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{56}> machine ddbcpu 0x39
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a98360, 4001513fbe0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{57}> machine ddbcpu 0x3a
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0aa8360, 4001513f3a0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{58}> machine ddbcpu 0x3b
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0ab8360, 4001513ee20, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{59}> machine ddbcpu 0x3c
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0ac8360, 4001513e5e0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{60}> machine ddbcpu 0x3d
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0ad8360, 4001513e320, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{61}> machine ddbcpu 0x3e
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0ae8360, 4001513e060, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{62}> machine ddbcpu 0x3f
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0af8360, 4001513e8a0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{63}> show panic
*cpu6: trap type 0x147 (*trap): pc=1668aec npc=1668af0 pstate=820006<PRIV,IE>
