On 3.12.2021. 20:35, Alexander Bluhm wrote: > Hi, > > After implementing MP safe refcounting for IPsec TDB, I wonder how > stable my diff for forwarding on multiple CPU is. > > Note that IPsec still has the workaround to disable multiple queues. > We do not have a mutex that protects the TDB fields yet. We have > only fixed the memory management. > > My goal is to get real MP pressure on the lower part of the IP > network stack. Only this will show remaining bugs.
Hi, with only plain forwarding it seems that everything works just fine, but with ipsec i'm getting this panic. i will leave ddb console active if something else will be needed r620-2# uvm_fault(0xffffffff8229d4e0, 0x137, 0, 2) -> e kernel: page fault trap, code=0 Stopped at ipsp_spd_lookup+0xa2f: movq %rax,0(%rcx) TID PID UID PRFLAGS PFLAGS CPU COMMAND 419237 67407 0 0x14000 0x200 0 softnet *157694 94649 0 0x14000 0x200 2K softnet ipsp_spd_lookup(fffffd80a4139800,2,14,2,0,0,5b815d966b14b44b,fffffd80a4139800) at ipsp_spd_lookup+0xa2f ip_output_ipsec_lookup(fffffd80a4139800,14,0,ffff800022c60228,0) at ip_output_ipsec_lookup+0x4c ip_output(fffffd80a4139800,0,ffff800022c603e8,1,0,0,3ada3367ffb43fe1) at ip_output+0x39d ip_forward(fffffd80a4139800,ffff800000087048,fffffd8394511078,0) at ip_forward+0x26a ip_input_if(ffff800022c60528,ffff800022c60534,4,0,ffff800000087048) at ip_input_if+0x353 ipv4_input(ffff800000087048,fffffd80a4139800) at ipv4_input+0x39 ether_input(ffff800000087048,fffffd80a4139800) at ether_input+0x3aa if_input_process(ffff800000087048,ffff800022c60618) at if_input_process+0x92 ifiq_process(ffff800000087458) at ifiq_process+0x69 taskq_thread(ffff80000002f080) at taskq_thread+0x81 end trace frame: 0x0, count: 5 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{2}> ddb{2}> ps PID TID PPID UID S FLAGS WAIT COMMAND 84832 15199 94097 68 3 0x10 netlock isakmpd 94097 209788 1 0 3 0x80 netio isakmpd 15214 351483 1 0 3 0x100083 ttyin ksh 12905 119865 1 0 3 0x100098 poll cron 84388 106290 61660 95 3 0x100092 kqread smtpd 49054 464016 61660 103 3 0x100092 kqread smtpd 24623 388600 61660 95 3 0x100092 kqread smtpd 25783 71877 61660 95 3 0x100092 kqread smtpd 44044 451515 61660 95 3 0x100092 kqread smtpd 74629 163500 61660 95 3 0x100092 kqread smtpd 61660 91566 1 0 3 0x100080 kqread smtpd 44941 436617 1 0 3 0x88 poll sshd 63116 236112 1 0 3 0x100080 poll ntpd 67685 455461 13478 83 3 0x100092 poll ntpd 13478 271069 1 83 3 0x100092 poll ntpd 4556 497201 93102 73 3 0x100090 kqread syslogd 93102 715 1 0 3 0x100082 netio syslogd 75807 17005 0 0 3 0x14200 bored smr 19891 296875 0 0 3 0x14200 pgzero zerothread 85805 493633 0 0 3 0x14200 aiodoned aiodoned 30502 291595 0 0 3 0x14200 syncer update 70346 429811 0 0 3 0x14200 cleaner cleaner 2149 206582 0 0 3 0x14200 reaper reaper 34585 457270 0 0 3 0x14200 pgdaemon pagedaemon 68729 397903 0 0 3 0x14200 usbtsk usbtask 65328 269730 0 0 3 0x14200 usbatsk usbatsk 69367 498053 0 0 3 0x40014200 acpi0 acpi0 5156 405582 0 0 7 0x40014200 idle5 53211 338561 0 0 7 0x40014200 idle4 19386 131626 0 0 7 0x40014200 idle3 22779 421169 0 0 3 0x40014200 idle2 27190 363359 0 0 7 0x40014200 idle1 1080 80711 0 0 3 0x14200 bored sensors 67407 419237 0 0 7 0x14200 softnet 34617 10042 0 0 3 0x14200 netlock softnet 50133 404397 0 0 3 0x14200 netlock softnet *94649 157694 0 0 7 0x14200 softnet 51030 188067 0 0 3 0x14200 bored systqmp 46228 294212 0 0 3 0x14200 bored systq 62154 451682 0 0 2 0x40014200 softclock 34808 227380 0 0 3 0x40014200 idle0 1 24458 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{2}> trace /t 0t157694 ffff800000087048(1,ffffffff8122bc39,ffff800022c60550,ffff800022c60528,ffff80002 2c60534,4) at 0xffff800000087048 __kernel_phys_end() at 0x22c605c0 end trace frame: 0x0, count: -2 ddb{2}> ddb{2}> mach ddbcpu 0 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffffffff8218eff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 pool_cache_get(ffffffff8229ee58) at pool_cache_get+0x19 pool_get(ffffffff8229ee58,2) at pool_get+0x5e m_clget(0,2,802) at m_clget+0x1b4 ixgbe_get_buf(ffff80000008a000,6e) at ixgbe_get_buf+0xa3 ixgbe_rxfill(ffff80000008a000) at ixgbe_rxfill+0xae ixgbe_queue_intr(ffff800000085400) at ixgbe_queue_intr+0x4f intr_handler(ffff800022c71920,ffff800000078480) at intr_handler+0x6e Xintr_ioapic_edge18_untramp() at Xintr_ioapic_edge18_untramp+0x18f _kernel_lock() at _kernel_lock+0xa6 ether_resolve(ffff8000000a0048,fffffd80a3bd3100,ffff800022c71b08,fffffd83945114 68,ffff800022c71a88) at ether_resolve+0x1a8 ether_output(ffff8000000a0048,fffffd80a3bd3100,ffff800022c71b08,fffffd839451146 8) at ether_output+0x2c end trace frame: 0xffff800022c71be0, count: 0 ddb{0}> ddb{0}> mach ddbcpu 1 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff800022409ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 acpicpu_idle() at acpicpu_idle+0x281 sched_idle(ffff800022409ff0) at sched_idle+0x27e end trace frame: 0x0, count: 10 ddb{1}> ddb{1}> mach ddbcpu 3 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff80002241bff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 acpicpu_idle() at acpicpu_idle+0x281 sched_idle(ffff80002241bff0) at sched_idle+0x27e end trace frame: 0x0, count: 10 ddb{3}> ddb{3}> mach ddbcpu 4 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff800022424ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 acpicpu_idle() at acpicpu_idle+0x281 sched_idle(ffff800022424ff0) at sched_idle+0x27e end trace frame: 0x0, count: 10 ddb{4}> ddb{4}> mach ddbcpu 5 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff80002242dff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 acpicpu_idle() at acpicpu_idle+0x281 sched_idle(ffff80002242dff0) at sched_idle+0x27e end trace frame: 0x0, count: 10 ddb{5}> ddb{2}> show tdb 0xffffffff81cfe23f: 000041bd (unknown address family)->(unknown address family) :15 #-1958084609 74f68445 ddb{2}> show tdb /f 0xffffffff81cfe23f tdb at 0xffffffff81cfe23f hnext: 0x130808b49018948 dnext: 0x138888b490000 snext: 0x3880c74901894800 inext: 0x49ffffffff000001 onext: 0xffff0000013080c7 xform: 0x8b4cffa5bd2be800 refcnt: -1958084609 encalgxform: 0x11080c7498845 authalgxform: 0x80c7490000000000 compalgxform: 0xf8 flags: 74f68445<UNIQUE,BYTES,DELETED,SOFT_FIRSTUSE,UDPENCAP,PFSYNC ,ESN> seq: -34635 exp_allocations: -2133708289 soft_allocations: 1207998333 cur_allocations: 949306639 exp_bytes: -53887557573804033 soft_bytes: -7572652700634382337 cur_bytes: -8986685076615987194 exp_timeout: 1274901163972733 soft_timeout: -1044682726943537152 established: -23376416473088 first_use: 1207830954720511 soft_first_use: -2045619644365824 exp_first_use: -956873529326089985 last_used: -528463706791608320 last_marked: -5151950549254864897 cryptoid: 434571561984018687 tdb_spi: 000041bd amxkeylen: 13 emxkeylen: 0 ivlen: 49285 sproto: 15 wnd: 133 satype: 76 updates: 248 dst: (unknown address family) src: (unknown address family) amxkey: 0x110b88b49ffffff emxkey: 0x4cffa5bae2e80000 rpl: 76702787085813131 ids: 0xf2b8809845 ids_swapped: -578482432 mtu: 1224736758 mtutimeout: 264675222111627 udpencap_port: 0 tag: 4073 tap: 1006632963 rdomain: 1012364293 rdomain_post: -1048244478