On May 12, 2023, at 10:26 AM, Martin Pieuchot <m...@openbsd.org> wrote:
>
> On 09/05/23(Tue) 20:02, Kurt Miller wrote:
>> While building devel/jdk/1.8 on May 3rd snapshot I noticed the build freezing
>> and processes getting stuck like ps. After enabling ddb.console I was able to
>> reproduce the livelock and capture cpu traces. Dmesg at the end.
>> Let me know if more information is needed as this appears to be rather
>> reproducible on my T4-1.
>
> It seems that all CPUs are waiting for the KERNEL_LOCK(). Doing ps /o
> in ddb(4) should show us which CPU is currently holding it. I can't
> figure it out just by looking at the traces below.
>
I don’t think this will be helpful. I reproduced the deadlock with the May 10
snapshot, broke into ddb and while doing the cpu traces hit a ddb panic while
Switching to cpu 18. ps /o output at end. I’ll try again to get a clean set of
traces.
login: Stopped at db_enter+0x8: nop
ddb{0}> bt
vcons_softintr(400151b8a00, 1c940b8, 20, 400e230c870, 11b5d40, 6) at
vcons_softintr+0x3c
intr_handler(2017ec8, 400151afa00, 2422a40a, 1c82298, 0, 6) at intr_handler+0x50
sparc_intr_retry(0, 0, 17cc340, 9e6475cc0c, 1c00, 12) at sparc_intr_retry+0x5c
cpu_idle_cycle(1c94098, 2018000, 184bdd0, 1c82298, 0, 19a6a30) at
cpu_idle_cycle+0x2c
sched_idle(2018360, 4001516a000, 17cc340, 0, 1c56e2e, 3b9ac800) at
sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{0}> machine ddbcpu 0x01
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{1}> bt
sched_idle(400e0718360, 4001516bb80, 17cc340, 0, 0, 3b9ac800) at
sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{1}> machine ddbcpu 0x2
Stopped at sun4v_send_ipi+0x88: bne,pn sun4v_send_ipi+0x98
ddb{2}> bt
setrunnable(40014c2b6b0, 1c57ad0, 18f1a58, 3, 1f63b9, 6) at setrunnable+0xe4
wakeup_proc(1, 400e2310350, 0, 2e4, 3, 6) at wakeup_proc+0xa0
wakeup_n(400e2310350, ffffffffffffffff, 0, 3ec772000, ffffffffffffffff,
1d1bf50) at wakeup_n+0x94
uvmfault_unlockall(400e5ff3cd0, 0, 400e230c870, 400e5ff3a00, 24, 4) at
uvmfault_unlockall+0xc
uvm_fault_lower(0, 400e5ff3d08, 40010cb0510, 40010cb0558, 1c04000, 1c7b2f8) at
uvm_fault_lower+0x880
uvm_fault(0, 400e5ff3cd0, 0, 400e230c870, 11b5d40, 1) at uvm_fault+0x1bc
text_access_fault(400e5ff3ed0, 9, 9ea37ea2e4, 0, 0, 0) at
text_access_fault+0x114
sun4v_texttrap(9ef623ffc0, 52a, 9e6475cc08, 9e6475cc0c, 0, 2b) at
sun4v_texttrap+0x1fc
ddb{2}> machine ddbcpu 0x3
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{3}> bt
sched_idle(400e0738360, 4001515d350, 17cc340, 1c00, 1825df8, 3b9ac800) at
sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{3}> machine ddbcpu 0x4
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{4}> bt
sched_idle(400e0748360, 4001515c590, 17cc340, 1c00, 1825df8, 3b9ac800) at
sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{4}> machine ddbcpu 0x5
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{5}> bt
sched_idle(400e0758360, 4001515c010, 17cc340, 400e0f11b78, 1825df8, 3b9ac800)
at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{5}> machine ddbcpu 0x6
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{6}> bt
sched_idle(400e0768360, 4001515c2d0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{6}> machine ddbcpu 0x7
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{7}> bt
sched_idle(400e0778360, 4001515c850, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{7}> machine ddbcpu 0x8
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{8}> bt
sched_idle(400e0788360, 4001515cb10, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{8}> machine ddbcpu 0x9
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{9}> bt
sched_idle(400e0798360, 4001515cdd0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{9}> machine ddbcpu 0xa
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{10}> b
sync crash dump halt reboot poweroff
ddb{10}> bt
sched_idle(400e07a8360, 4001515d090, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{10}> machine ddbcpu 0xb
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{11}> bt
sched_idle(400e07b8360, 4001515d610, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{11}> machine ddbcpu 0xc
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{12}> bt
sched_idle(400e07c8360, 4001515db90, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{12}> machine ddbcpu 0xd
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{13}> bt
sched_idle(400e07d8360, 40015157ba0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{13}> machine ddbcpu 0xe
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{14}> b
sync crash dump halt reboot poweroff
ddb{14}> bt
sched_idle(400e07e8360, 400151578e0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{14}> machine ddbcpu 0xf
Stopped at __mp_lock+0x64: nop
ddb{15}> bt
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{15}> machine ddbcpu 0x10
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{16}> bt
sched_idle(400e0808360, 40015156020, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{16}> machine ddbcpu 0x11
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
ddb{17}> bt
sched_idle(400e0818360, 400151562e0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{17}> machine ddbcpu 0x12
pSantico: pmupteex d0x 40 0e 08 d8 82 0 no t he ld i n cl oc kq u eu e_ ne
xt
at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
TID PID UID PRFLAGS PFLAGS CPU COMMAND
389763 59102 55 0x200003 0x4000000 2 javac
sched_idle(400e0828360, 400151565a0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
https://www.openbsd.org/ddb.html describes the minimum info required in bug
reports. Insufficient info makes it difficult to find and fix bugs.
ddb{18}> bt
sched_idle(400e0828360, 400151565a0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{18}> machine ddbcpu 0x13
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0838360, 40015156860, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{19}> bt
sched_idle(400e0838360, 40015156860, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{19}> machine ddbcpu 0x14
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0848360, 40015156b20, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{20}> machine ddbcpu 0x15
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0858360, 400151570a0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{21}> machine ddbcpu 0x16
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0868360, 40015157360, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{22}> machine ddbcpu 0x17
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0878360, 40015157620, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{23}> machine ddbcpu 0x18
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0888360, 40015151bb0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{24}> machine ddbcpu 0x19
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0898360, 400151518f0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{25}> machine ddbcpu 0x1a
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e08a8360, 40015151370, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{26}> machine ddbcpu 0x1b
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e08b8360, 400151510b0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{27}> machine ddbcpu 0x1c
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e08c8360, 40015150df0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{28}> machine ddbcpu 0x1d
Stopped at db_enter+0x8: nop
clockqueue_next(400e08d87e0, 2152ca4f32d5, 17ceb68, 9e6475cc0c, 0, 12) at
clockqueue_next+0x70
clockintr_dispatch(400e08d7ec8, 400e08d8000, 184bdd0, 1c82298, f02662a4,
19a6a30) at clockintr_dispatch+0xb8
sys_tickintr(1, 400151502f0, 0, 0, 0, 0) at sys_tickintr+0x8
intr_handler(400e08d7ec8, 400e08d88d8, 2440a808, 0, 1c06000, 0) at
intr_handler+0x50
sparc_intr_retry(1c11958, 6, 1c11958, 0, 0, 4) at sparc_intr_retry+0x5c
db_enter_ddb(1c8a000, 400e5ff3d08, 400e5ff3bd0, 0, 1c04000, 1c7b2f8) at
db_enter_ddb+0xbc
db_ktrap(101, 400e0e4db50, 20, 400e230c870, 11b5d40, 1) at db_ktrap+0x104
trap(400e0e4db50, 101, 11b1100, 44820006, 0, 6) at trap+0x2cc
Lslowtrap_reenter(0, 0, 24229eed, 9e6475cc0c, 0, 12) at Lslowtrap_reenter+0xf8
cpu_idle_cycle(1c94098, 400e08d8000, 184bdd0, 1c82298, f02662a4, 19a6a30) at
cpu_idle_cycle+0x2c
sched_idle(400e08d8360, 400151502f0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{29}> machine ddbcpu 0x1e
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e08e8360, 40015150030, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{30}> machine ddbcpu 0x1f
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e08f8360, 400151505b0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{31}> machine ddbcpu 0x20
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0908360, 40015150870, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{32}> machine ddbcpu 0x21
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0918360, 40015150b30, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{33}> machine ddbcpu 0x22
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0928360, 40015151630, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{34}> machine ddbcpu 0x23
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0938360, 4001514b900, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{35}> machine ddbcpu 0x24
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0948360, 4001514b640, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{36}> machine ddbcpu 0x25
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0958360, 4001514ae00, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{37}> machine ddbcpu 0x26
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0968360, 4001514ab40, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{38}> machine ddbcpu 0x27
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0978360, 4001514a880, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{39}> machine ddbcpu 0x28
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0988360, 4001514a5c0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{40}> machine ddbcpu 0x29
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0998360, 4001514a300, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{41}> machine ddbcpu 0x2a
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e09a8360, 4001514a040, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{42}> machine ddbcpu 0x2b
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e09b8360, 4001514b0c0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{43}> machine ddbcpu 0x2c
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e09c8360, 4001514b380, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{44}> d
Not set.
ddb{44}> machine ddbcpu 0x2d
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e09d8360, 4001514bbc0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{45}> bt
sched_idle(400e09d8360, 4001514bbc0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{45}> machine ddbcpu 0x2e
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e09e8360, 40015145bd0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{46}> machine ddbcpu 0x2f
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e09f8360, 40015145910, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{47}> machine ddbcpu 0x30
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a08360, 40015145650, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{48}> machine ddbcpu 0x31
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a18360, 40015144e10, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{49}> machine ddbcpu 0x32
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a28360, 40015144b50, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{50}> machine ddbcpu 0x33
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a38360, 40015144050, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{51}> machine ddbcpu 0x34
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a48360, 40015144310, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{52}> machine ddbcpu 0x35
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a58360, 400151445d0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{53}> machine ddbcpu 0x36
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a68360, 40015144890, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{54}> machine ddbcpu 0x37
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a78360, 400151450d0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{55}> machine ddbcpu 0x38
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a88360, 40015145390, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{56}> machine ddbcpu 0x39
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0a98360, 4001513fbe0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{57}> machine ddbcpu 0x3a
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0aa8360, 4001513f660, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{58}> machine ddbcpu 0x3b
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0ab8360, 4001513ee20, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{59}> machine ddbcpu 0x3c
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0ac8360, 4001513e060, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{60}> machine ddbcpu 0x3d
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0ad8360, 4001513e320, 17cc340, e80, 44, 3b9ac800) at
sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{61}> machine ddbcpu 0x3e
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0ae8360, 4001513e5e0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{62}> machine ddbcpu 0x3f
Stopped at cpu_idle_cycle+0x44: and %g1, -0x3, %g1
sched_idle(400e0af8360, 4001513e8a0, 0, 0, 0, 0) at sched_idle+0x158
proc_trampoline(0, 0, 0, 0, 0, 0) at proc_trampoline+0x14
ddb{63}> ps /o
TID PID UID PRFLAGS PFLAGS CPU COMMAND
389763 59102 55 0x200003 0x4000000 2 javac
ddb{63}> show panic
*cpu29: mutex 0x400e08d8820 not held in clockqueue_next
