Hi,
another crash...
kernel: privileged instruction fault trap, code=0
Stopped at x86_ipi_handler+0x68: shll %cl,%eax
ddb{2}> show panic
the kernel did not panic
ddb{2}> trace
x86_ipi_handler() at x86_ipi_handler+0x68
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x27
acpicpu_idle() at acpicpu_idle+0x131
sched_idle(ffff80002d4b7ff0) at sched_idle+0x298
end trace frame: 0x0, count: -4
ddb{2}> show register
rdi 0
rsi 0
rbp 0xffff80002d6a1160
rbx 0xffffffff8274f328 ipifunc+0x18
rdx 0
rcx 0x3
rax 0x1
r8 0
r9 0
r10 0
r11 0x2bdfc7a9cf2bf678
r12 0x3
r13 0
r14 0xffff80002d4b7ff0
r15 0x40
rip 0xffffffff817067f8 x86_ipi_handler+0x68
cs 0x8
rflags 0x10202 __ALIGN_SIZE+0xf202
rsp 0xffff80002d6a1130
ss 0x10
x86_ipi_handler+0x68: shll %cl,%eax
ddb{2}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
8492 243348 1 0 3 0x100083 ttyin getty
70872 302086 1 0 3 0x100098 kqread cron
62053 331002 1 0 3 0x80 ugenrintr apcupsd
62053 93517 1 0 3 0x4000088 sigwait apcupsd
62053 28200 1 0 3 0x4000080 netacc apcupsd
15763 328337 1 99 3 0x1100090 kqread sndiod
38180 229683 1 110 3 0x100090 kqread sndiod
31304 148057 63862 95 3 0x1100092 kqread smtpd
66174 206858 63862 103 3 0x1100092 kqread smtpd
66648 18442 63862 95 3 0x1100092 kqread smtpd
74037 501012 63862 95 3 0x100092 kqread smtpd
57115 211395 63862 95 3 0x1100092 kqread smtpd
72085 343660 63862 95 3 0x1100092 kqread smtpd
63862 109323 1 0 3 0x100080 kqread smtpd
29631 384396 1 77 3 0x1100090 kqread dhcpd
82518 421719 1 0 3 0x88 kqread sshd
29221 79980 74757 68 3 0x1000090 kqread isakmpd
74757 97916 1 0 3 0x80 sbwait isakmpd
17008 198783 1 0 3 0x100080 kqread ntpd
11490 248441 7899 83 3 0x100092 kqread ntpd
7899 240306 1 83 3 0x1100092 kqread ntpd
35443 94254 1 53 3 0x1000090 kqread unbound
63299 188106 26777 73 3 0x1100090 kqread syslogd
26777 424008 1 0 3 0x100082 sbwait syslogd
12308 448270 1 0 3 0x100080 kqread resolvd
74922 231097 4647 77 3 0x100092 kqread dhcpleased
46525 390077 4647 77 3 0x100092 kqread dhcpleased
4647 4557 1 0 3 0x80 kqread dhcpleased
7696 400067 59717 115 3 0x100092 kqread slaacd
30367 283456 59717 115 3 0x100092 kqread slaacd
59717 98556 1 0 3 0x100080 kqread slaacd
99966 166241 0 0 3 0x14200 bored smr
85628 113744 0 0 3 0x14200 pgzero zerothread
62434 427146 0 0 3 0x14200 aiodoned aiodoned
4869 387969 0 0 3 0x14200 syncer update
25255 365094 0 0 3 0x14200 cleaner cleaner
64672 273420 0 0 3 0x14200 reaper reaper
56364 438602 0 0 3 0x14200 pgdaemon pagedaemon
1926 390504 0 0 3 0x14200 mmctsk sdmmc0
27874 111638 0 0 3 0x14200 usbtsk usbtask
68423 461158 0 0 3 0x14200 usbatsk usbatsk
81699 445126 0 0 3 0x40014200 acpi0 acpi0
68350 172014 0 0 7 0x40014200 idle3
*53748 68447 0 0 7 0x40014200 idle2
19205 228943 0 0 7 0x40014200 idle1
36392 233525 0 0 3 0x14200 bored sensors
24164 256389 0 0 3 0x14200 bored softnet3
78493 67841 0 0 3 0x14200 bored softnet2
85105 303392 0 0 3 0x14200 bored softnet1
62948 405515 0 0 2 0x14200 softnet0
17409 302245 0 0 3 0x14200 bored systqmp
85175 190072 0 0 3 0x14200 bored systq
78099 348947 0 0 3 0x14200 tmoslp softclockmp
10691 186666 0 0 3 0x40014200 tmoslp softclock
80798 391219 0 0 7 0x40014200 idle0
1 490462 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{2}> mach ddbcpu 0
Stopped at x86_ipi_db+0x16: leave
ddb{0}> mach ddbcpu 1
Stopped at x86_ipi_db+0x16: leave
ddb{1}> mach ddbcpu 2
Stopped at x86_ipi_handler+0x68: shll %cl,%eax
ddb{2}> mach ddbcpu 3
Stopped at x86_ipi_db+0x16: leave
ddb{3}> dmesg
OpenBSD 7.6 (GENERIC.MP) #0: Thu Jan 9 07:32:40 MST 2025
[email protected]:/usr/src/sys/arch/amd64/compile/GENERIC.
MP
real mem = 4259897344 (4062MB)
avail mem = 4107632640 (3917MB)
random: good seed from bootblocks
mpath0 at root
scsibus0 at mpath0: 256 targets
mainbus0 at root
bios0 at mainbus0: SMBIOS rev. 3.0 @ 0xcfe92040 (13 entries)
bios0: vendor coreboot version "v4.17.0.1" date 06/22/2022
bios0: PC Engines apu4
acpi0 at bios0: ACPI 6.0
acpi0: sleep states S0 S1 S4 S5
acpi0: tables DSDT FACP SSDT MCFG TPM2 APIC HEST SSDT SSDT DRTM HPET
acpi0: wakeup devices PBR4(S4) PBR5(S4) PBR6(S4) PBR7(S4) PBR8(S4) UOH1(S3) UOH
2(S3) UOH3(S3) UOH4(S3) UOH5(S3) UOH6(S3) XHC0(S4)
acpitimer0 at acpi0: 3579545 Hz, 32 bits
acpimcfg0 at acpi0
acpimcfg0: addr 0xf8000000, bus 0-63
acpimadt0 at acpi0 addr 0xfee00000: PC-AT compat
cpu0 at mainbus0: apid 0 (boot processor)
cpu0: AMD GX-412TC SOC, 998.18 MHz, 16-30-01, patch 07030105
cpu0: cpuid 1 edx=178bfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE
,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT> ecx=36d8220b<SSE3,PCLMUL,MWAI
T,SSSE3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C>
cpu0: cpuid 6 eax=4<ARAT> ecx=1<EFFFREQ>
cpu0: cpuid 7.0 ebx=8<BMI1>
cpu0: cpuid d.1 eax=1<XSAVEOPT>
cpu0: cpuid 80000001 edx=2fd3fbff<NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG> ecx=1d403
7ff<LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,IBS,SKINIT,TOPEXT
,DBKP,PERFTSC,PCTRL3>
cpu0: cpuid 80000007 edx=33d9<HWPSTATE,ITSC>
cpu0: 32KB 64b/line 8-way D-cache, 32KB 64b/line 2-way I-cache, 2MB 64b/line 16
-way L2 cache
cpu0: smt 0, core 0, package 0
mtrr: Pentium Pro MTRR support, 8 var ranges, 88 fixed ranges
cpu0: apic clock running at 99MHz
cpu0: mwait min=64, max=64, IBE
cpu1 at mainbus0: apid 1 (application processor)
cpu1: AMD GX-412TC SOC, 998.24 MHz, 16-30-01, patch 07030105
cpu1: smt 0, core 1, package 0
cpu2 at mainbus0: apid 2 (application processor)
cpu2: AMD GX-412TC SOC, 998.27 MHz, 16-30-01, patch 07030105
cpu2: smt 0, core 2, package 0
cpu3 at mainbus0: apid 3 (application processor)
cpu3: AMD GX-412TC SOC, 998.39 MHz, 16-30-01, patch 07030105
cpu3: smt 0, core 3, package 0
ioapic0 at mainbus0: apid 4 pa 0xfec00000, version 21, 24 pins
ioapic1 at mainbus0: apid 5 pa 0xfec20000, version 21, 32 pins
acpihpet0 at acpi0: 14318180 Hz
acpiprt0 at acpi0: bus 0 (PCI0)
acpiprt1 at acpi0: bus 1 (PBR4)
acpiprt2 at acpi0: bus 2 (PBR5)
acpiprt3 at acpi0: bus 3 (PBR6)
acpiprt4 at acpi0: bus 4 (PBR7)
acpiprt5 at acpi0: bus -1 (PBR8)
acpicpu0 at acpi0: C2(0@400 io@0x1771), C1(@1 halt!), PSS
acpicpu1 at acpi0: C2(0@400 io@0x1771), C1(@1 halt!), PSS
acpicpu2 at acpi0: C2(0@400 io@0x1771), C1(@1 halt!), PSS
acpicpu3 at acpi0: C2(0@400 io@0x1771), C1(@1 halt!), PSS
acpipci0 at acpi0 PCI0: 0x00000000 0x00000011 0x00000001
acpicmos0 at acpi0
com0 at acpi0 COM1 addr 0x3f8/0x8 irq 4: ns16550a, 16 byte fifo
com0: console
com1 at acpi0 COM2 addr 0x2f8/0x8 irq 3: ns16550a, 16 byte fifo
amdgpio0 at acpi0 GPIO uid 0 addr 0xfed81500/0x300 irq 7, 184 pins
"PRP0001" at acpi0 not configured
"PRP0001" at acpi0 not configured
"PRP0001" at acpi0 not configured
"PRP0001" at acpi0 not configured
"PRP0001" at acpi0 not configured
"PRP0001" at acpi0 not configured
"BOOT0000" at acpi0 not configured
acpitz0 at acpi0: critical temperature is 115 degC
cpu0: 998 MHz: speeds: 1000 800 600 MHz
pci0 at mainbus0 bus 0
pchb0 at pci0 dev 0 function 0 "AMD 16h Root Complex" rev 0x00
vendor "AMD", unknown product 0x1567 (class system subclass IOMMU, rev 0x00) at
pci0 dev 0 function 2 not configured
pchb1 at pci0 dev 2 function 0 "AMD 16h Host" rev 0x00
ppb0 at pci0 dev 2 function 1 "AMD 16h PCIE" rev 0x00: msi
pci1 at ppb0 bus 1
em0 at pci1 dev 0 function 0 "Intel I211" rev 0x03: msi, address 00:0d:b9:59:e0
:e4
ppb1 at pci0 dev 2 function 2 "AMD 16h PCIE" rev 0x00: msi
pci2 at ppb1 bus 2
em1 at pci2 dev 0 function 0 "Intel I211" rev 0x03: msi, address 00:0d:b9:59:e0
:e5
ppb2 at pci0 dev 2 function 3 "AMD 16h PCIE" rev 0x00: msi
pci3 at ppb2 bus 3
em2 at pci3 dev 0 function 0 "Intel I211" rev 0x03: msi, address 00:0d:b9:59:e0
:e6
ppb3 at pci0 dev 2 function 4 "AMD 16h PCIE" rev 0x00: msi
pci4 at ppb3 bus 4
em3 at pci4 dev 0 function 0 "Intel I211" rev 0x03: msi, address 00:0d:b9:59:e0
:e7
ccp0 at pci0 dev 8 function 0 "AMD 16h Crypto" rev 0x00: msix
xhci0 at pci0 dev 16 function 0 "AMD Bolton xHCI" rev 0x11: msix, xHCI 1.0
usb0 at xhci0: USB revision 3.0
uhub0 at usb0 configuration 1 interface 0 "AMD xHCI root hub" rev 3.00/1.00 add
r 1
ahci0 at pci0 dev 17 function 0 "AMD Hudson-2 SATA" rev 0x40: apic 4 int 19, AH
CI 1.3
ahci0: port 0: 6.0Gb/s
scsibus1 at ahci0: 32 targets
sd0 at scsibus1 targ 0 lun 0: <ATA, Hoodisk SSD, SBFM> t10.ATA_Hoodisk_SSD_L7DT
C7A11208345_
sd0: 15272MB, 512 bytes/sector, 31277232 sectors, thin
ehci0 at pci0 dev 18 function 0 "AMD Hudson-2 USB2" rev 0x39: apic 4 int 18
usb1 at ehci0: USB revision 2.0
uhub1 at usb1 configuration 1 interface 0 "AMD EHCI root hub" rev 2.00/1.00 add
r 1
ehci1 at pci0 dev 19 function 0 "AMD Hudson-2 USB2" rev 0x39: apic 4 int 18
usb2 at ehci1: USB revision 2.0
uhub2 at usb2 configuration 1 interface 0 "AMD EHCI root hub" rev 2.00/1.00 add
r 1
piixpm0 at pci0 dev 20 function 0 "AMD Hudson-2 SMBus" rev 0x42: SMI
iic0 at piixpm0
iic1 at piixpm0
iic1: addr 0x4c 3e=00 48=00 4a=00 4e=00 fc=00 fe=00 words 00=ffff 01=ffff 02=ff
ff 03=ffff 04=ffff 05=ffff 06=ffff 07=ffff
pcib0 at pci0 dev 20 function 3 "AMD Hudson-2 LPC" rev 0x11
sdhc0 at pci0 dev 20 function 7 "AMD Bolton SD/MMC" rev 0x01: apic 4 int 16
sdhc0: SDHC 2.00, 50 MHz base clock
sdmmc0 at sdhc0: 4-bit, sd high-speed, mmc high-speed, dma
pchb2 at pci0 dev 24 function 0 "AMD 16h Link Cfg" rev 0x00
pchb3 at pci0 dev 24 function 1 "AMD 16h Address Map" rev 0x00
pchb4 at pci0 dev 24 function 2 "AMD 16h DRAM Cfg" rev 0x00
km0 at pci0 dev 24 function 3 "AMD 16h Misc Cfg" rev 0x00
pchb5 at pci0 dev 24 function 4 "AMD 16h CPU Power" rev 0x00
pchb6 at pci0 dev 24 function 5 "AMD 16h Misc Cfg" rev 0x00
isa0 at pcib0
isadma0 at isa0
com2 at isa0 port 0x3e8/8 irq 5: ns16550a, 16 byte fifo
pcppi0 at isa0 port 0x61
spkr0 at pcppi0
lpt0 at isa0 port 0x378/4 irq 7
intr_establish: pic ioapic0 pin 7: can't share type 3 with 2
wbsio0 at isa0 port 0x2e/2: NCT5104D rev 0x53
vmm0 at mainbus0: SVM/RVI
ugen0 at uhub0 port 3 "American Power Conversion Back-UPS CS 350 FW:807.q10 .I U
SB FW:q10" rev 1.10/0.06 addr 2
uhub3 at uhub1 port 1 configuration 1 interface 0 "Advanced Micro Devices Hub" r
ev 2.00/0.18 addr 2
uhub4 at uhub2 port 1 configuration 1 interface 0 "Advanced Micro Devices Hub" r
ev 2.00/0.18 addr 2
vscsi0 at root
scsibus2 at vscsi0: 256 targets
softraid0 at root
scsibus3 at softraid0: 256 targets
root on sd0a (cbb37b39d1463c87.a) swap on sd0b dump on sd0b
On Sat, 18 Jan 2025 16:53:11 +0100
Alexander Bluhm <[email protected]> wrote:
> On Sat, Jan 18, 2025 at 02:54:41PM +0100, Radek wrote:
> > Hi,
> > this is another crash, including the output that was before the ddb> prompt:
> >
> > panic: mtx 0xffffffff828b6010: locking against myself
> > Stopped at db_enter+0x14: popq %rbp
> > TID PID UID PRFLAGS PFLAGS CPU COMMAND
> > *307531 49394 0 0x14000 0x200 2K softnet0
> > db_enter() at db_enter+0x14
> > panic(ffffffff823bc733) at panic+0xdd
> > mtx_enter_try(ffffffff828b6010) at mtx_enter_try+0xd1
> > mtx_enter(ffffffff828b6010) at mtx_enter+0x35
> > pool_put(ffffffff828b6010,fffffd8125ad6d20) at pool_put+0x60
> > esp_output(fffffd80cd286e00,ffff800012f7e628,14,9) at esp_output+0x899
> > ipsp_process_packet(fffffd80cd286100,ffff800012f7e628,2,0) at
> > ipsp_process_pack
> > et+0x418
> > ip_output_ipsec_send(ffff800012f7e628,fffffd80cd286100,ffff80002d67cd78,1)
> > at i
> > p_output_ipsec_send+0x2a0
> > ip_output(fffffd80cd286100,0,ffff80002d67cd78,1,0,0,ce37a086de3f581f) at
> > ip_out
> > put+0x82b
> > ip_forward(fffffd80cd286100,ffff8000000b2048,ffff80002d67cd78,1) at
> > ip_forward+
> > 0x1e1
> > ip_input_if(ffff80002d67ce58,ffff80002d67ce64,7c,0,ffff8000000b2048) at
> > ip_inpu
> > t_if+0x3fa
> > ipv4_input(ffff8000000b2048,fffffd80cd286100) at ipv4_input+0x38
> > ether_input(ffff8000000b2048,fffffd80cd286100) at ether_input+0x3df
> > if_input_process(ffff8000000b2048,ffff80002d67cf48) at if_input_process+0x78
> > end trace frame: 0xffff80002d67cf90, count: 0
> > https://www.openbsd.org/ddb.html describes the minimum info required in bug
> > reports. Insufficient info makes it difficult to find and fix bugs.
>
> It crashes here:
>
> /home/bluhm/openbsd/stable-7.6/src/sys/netinet/ip_esp.c:947
> 1c94: 48 89 df mov %rbx,%rdi
> 1c97: e8 00 00 00 00 callq 1c9c <esp_output+0x87c>
> 1c9c: 48 c7 44 24 f8 00 00 movq $0x0,0xfffffffffffffff8(%rsp)
> 1ca3: 00 00
> 1ca5: 83 f8 23 cmp $0x23,%eax
> 1ca8: 74 d6 je 1c80 <esp_output+0x860>
> 1caa: 41 89 c4 mov %eax,%r12d
> 1cad: 85 c0 test %eax,%eax
> 1caf: 75 60 jne 1d11 <esp_output+0x8f1>
> /home/bluhm/openbsd/stable-7.6/src/sys/netinet/ip_esp.c:959
> 1cb1: 48 89 df mov %rbx,%rdi
> 1cb4: e8 00 00 00 00 callq 1cb9 <esp_output+0x899>
> /home/bluhm/openbsd/stable-7.6/src/sys/netinet/ip_esp.c:962
> * 1cb9: 48 c7 44 24 f8 00 00 movq $0x0,0xfffffffffffffff8(%rsp)
> 1cc0: 00 00
> 1cc2: 4c 89 f7 mov %r14,%rdi
> 1cc5: 4c 89 fe mov %r15,%rsi
> 1cc8: e8 00 00 00 00 callq 1ccd <esp_output+0x8ad>
> 1ccd: 48 c7 44 24 f8 00 00 movq $0x0,0xfffffffffffffff8(%rsp)
> 1cd4: 00 00
> /home/bluhm/openbsd/stable-7.6/src/sys/netinet/ip_esp.c:963
>
> /home/bluhm/openbsd/stable-7.6/src/sys/netinet/ip_esp.c
> 947 while ((error = crypto_invoke(crp)) == EAGAIN) {
> 948 /* Reset the session ID */
> 949 if (tdb->tdb_cryptoid != 0)
> 950 tdb->tdb_cryptoid = crp->crp_sid;
> 951 }
> 952 if (error) {
> 953 DPRINTF("crypto error %d", error);
> 954 ipsecstat_inc(ipsec_noxform);
> 955 goto drop;
> 956 }
> 957
> 958 /* Release the crypto descriptors */
> * 959 crypto_freereq(crp);
> 960
> 961 /* Call the IPsec input callback. */
> 962 error = ipsp_process_done(m, tdb);
> 963 if (error)
> 964 espstat_inc(esps_outfail);
> 965 return (error);
>
> /home/bluhm/openbsd/stable-7.6/src/sys/kern/subr_pool.c:789
> 24a8: 49 83 be b0 00 00 00 cmpq $0x0,0xb0(%r14)
> 24af: 00
> 24b0: 74 0e je 24c0 <pool_put+0x50>
> 24b2: 49 83 be 50 01 00 00 cmpq $0x0,0x150(%r14)
> 24b9: 00
> 24ba: 0f 84 d0 01 00 00 je 2690 <pool_put+0x220>
> /home/bluhm/openbsd/stable-7.6/src/sys/kern/subr_pool.c:104
> 24c0: 49 8b 46 10 mov 0x10(%r14),%rax
> 24c4: 4c 8b 58 08 mov 0x8(%rax),%r11
> 24c8: 4c 89 f7 mov %r14,%rdi
> 24cb: e8 00 00 00 00 callq 24d0 <pool_put+0x60>
> /home/bluhm/openbsd/stable-7.6/src/sys/kern/subr_pool.c:797
> * 24d0: 48 c7 44 24 f8 00 00 movq $0x0,0xfffffffffffffff8(%rsp)
> 24d7: 00 00
> 24d9: 4c 89 f7 mov %r14,%rdi
> 24dc: 4c 89 fe mov %r15,%rsi
> 24df: e8 00 00 00 00 callq 24e4 <pool_put+0x74>
> /home/bluhm/openbsd/stable-7.6/src/sys/kern/subr_pool.c:799
>
> /home/bluhm/openbsd/stable-7.6/src/sys/kern/subr_pool.c
> 101 static inline void
> 102 pl_enter(struct pool *pp, union pool_lock *pl)
> 103 {
> * 104 pp->pr_lock_ops->pl_enter(pl);
> 105 }
> ...
> 788 #ifdef MULTIPROCESSOR
> 789 if (pp->pr_cache != NULL && TAILQ_EMPTY(&pp->pr_requests)) {
> 790 pool_cache_put(pp, v);
> 791 return;
> 792 }
> 793 #endif
> 794
> * 795 pl_enter(pp, &pp->pr_lock);
> 796
> 797 pool_do_put(pp, v);
> 798
> 799 pp->pr_nout--;
>
> The crypto pool is protect by a mutex with IPL_VM. The crypto
> descriptors are allocated and freed in the same function esp_output().
> I don't understand how this could go wrong.
>
> bluhm
>
--
Please do not CC me
Radek
