On Sat, Apr 27, 2013 at 10:17:32AM +0200, Ian FREISLICH wrote: > Hi > > I've been getting the following panic on recent current r249717. > Sadly the crashdump is useless. >
I just saw similar panic on 10-CURRENT r249588.
> Fatal trap 9: general protection fault while in kernel mode
> cpuid = 15; apic id = 0f
> instruction pointer = 0x20:0xffffffff80546fbc
> stack pointer = 0x28:0xffffff846b677770
> frame pointer = 0x28:0xffffff846b6777b0
> code segment = base 0x0, limit 0xfffff, type 0x1b
> = DPL 0, pres 1, long 1, def32 0, gran 1
> processor eflags = interrupt enabled, resume, IOPL = 0
> current process = 4361 (zabbix_agentd)
Hmm.. This is interests me. In my case, cf-agent was the current
process.
Backtrace of my panic follows. Any pointers on how to debug this
further would be appreciated.
Glen
Script started on Sat Apr 27 23:53:53 2013
root@orion:/usr/obj/usr/src/sys/ORION # kgdb ./kernel.debug /var/crash/vmcore.4
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "amd64-marcel-freebsd"...
Unread portion of the kernel message buffer:
Fatal trap 9: general protection fault while in kernel mode
cpuid = 1; apic id = 01
instruction pointer = 0x20:0xffffffff80736cec
stack pointer = 0x28:0xffffff81aad4e760
frame pointer = 0x28:0xffffff81aad4e7a0
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 78664 (cf-agent)
trap number = 9
panic: general protection fault
cpuid = 1
KDB: stack backtrace:
#0 0xffffffff80642a56 at kdb_backtrace+0x66
#1 0xffffffff80606eeb at panic+0x13b
#2 0xffffffff808e3b10 at trap_fatal+0x290
#3 0xffffffff808e4331 at trap+0x241
#4 0xffffffff808cdbb3 at calltrap+0x8
#5 0xffffffff807371d8 at in_pcb_lport+0x128
#6 0xffffffff8073745a at in_pcbbind_setup+0x16a
#7 0xffffffff80737d8e at in_pcbconnect_setup+0x71e
#8 0xffffffff80737df9 at in_pcbconnect_mbuf+0x59
#9 0xffffffff807bf29f at udp_connect+0x11f
#10 0xffffffff80680615 at kern_connectat+0x275
#11 0xffffffff80680731 at sys_connect+0x41
#12 0xffffffff808e32cb at amd64_syscall+0x63b
#13 0xffffffff808cde97 at Xfast_syscall+0xf7
Uptime: 3d19h38m52s
(ada0:ahcich0:0:0:0): FLUSHCACHE48. ACB: ea 00 00 00 00 40 00 00 00 00 00 00
(ada0:ahcich0:0:0:0): CAM status: CCB request is in progress
(ada0:ahcich0:0:0:0): Error 5, Retries exhausted
(ada0:ahcich0:0:0:0): Synchronize cache failed
(ada1:ahcich1:0:0:0): FLUSHCACHE48. ACB: ea 00 00 00 00 40 00 00 00 00 00 00
(ada1:ahcich1:0:0:0): CAM status: CCB request is in progress
(ada1:ahcich1:0:0:0): Error 5, Retries exhausted
(ada1:ahcich1:0:0:0): Synchronize cache failed
(ada2:ahcich4:0:0:0): FLUSHCACHE48. ACB: ea 00 00 00 00 40 00 00 00 00 00 00
(ada2:ahcich4:0:0:0): CAM status: CCB request is in progress
(ada2:ahcich4:0:0:0): Error 5, Retries exhausted
(ada2:ahcich4:0:0:0): Synchronize cache failed
(ada3:ahcich5:0:0:0): FLUSHCACHE48. ACB: ea 00 00 00 00 40 00 00 00 00 00 00
(ada3:ahcich5:0:0:0): CAM status: CCB request is in progress
(ada3:ahcich5:0:0:0): Error 5, Retries exhausted
(ada3:ahcich5:0:0:0): Synchronize cache failed
Dumping 1014 out of 6049 MB:..2%..12%..21%..32%..42%..51%..62%..71%..81%..92%
Reading symbols from /boot/kernel/zfs.ko.symbols...done.
Loaded symbols for /boot/kernel/zfs.ko.symbols
Reading symbols from /boot/kernel/opensolaris.ko.symbols...done.
Loaded symbols for /boot/kernel/opensolaris.ko.symbols
#0 doadump (textdump=<value optimized out>) at pcpu.h:231
231 __asm("movq %%gs:%1,%0" : "=r" (td)
(kgdb) bt
#0 doadump (textdump=<value optimized out>) at pcpu.h:231
#1 0xffffffff80606a56 in kern_reboot (howto=260) at
/usr/src/sys/kern/kern_shutdown.c:447
#2 0xffffffff80606ed5 in panic (fmt=<value optimized out>)
at /usr/src/sys/kern/kern_shutdown.c:754
#3 0xffffffff808e3b10 in trap_fatal (frame=0x9, eva=<value optimized out>)
at /usr/src/sys/amd64/amd64/trap.c:872
#4 0xffffffff808e4331 in trap (frame=0xffffff81aad4e6b0)
at /usr/src/sys/amd64/amd64/trap.c:605
#5 0xffffffff808cdbb3 in calltrap () at
/usr/src/sys/amd64/amd64/exception.S:228
#6 0xffffffff80736cec in in_pcblookup_local (pcbinfo=0xffffffff80dc9180, laddr=
{s_addr = 50374848}, lport=339, lookupflags=1, cred=0xfffffe016cdad100)
at /usr/src/sys/netinet/in_pcb.c:1438
#7 0xffffffff807371d8 in in_pcb_lport (inp=0xfffffe016c2fb7a8,
laddrp=0xffffff81aad4e860,
lportp=0xffffff81aad4e86e, cred=0xfffffe016cdad100, lookupflags=1)
at /usr/src/sys/netinet/in_pcb.c:457
#8 0xffffffff8073745a in in_pcbbind_setup (inp=0xfffffe016c2fb7a8, nam=0x0,
laddrp=0xffffff81aad4e900, lportp=0xffffff81aad4e90e,
cred=0xfffffe016cdad100)
at /usr/src/sys/netinet/in_pcb.c:615
#9 0xffffffff80737d8e in in_pcbconnect_setup (inp=0xfffffe016c2fb7a8,
nam=<value optimized out>, laddrp=0xffffff81aad4e9b8,
lportp=0xffffff81aad4e9be,
faddrp=0xffffff81aad4e9b4, fportp=0xffffff81aad4e9bc, oinpp=0x0,
cred=0xfffffe016cdad100)
at /usr/src/sys/netinet/in_pcb.c:1019
#10 0xffffffff80737df9 in in_pcbconnect_mbuf (inp=0xfffffe016c2fb7a8,
nam=<value optimized out>, cred=<value optimized out>, m=0x0)
at /usr/src/sys/netinet/in_pcb.c:645
#11 0xffffffff807bf29f in udp_connect (so=0xfffffe016ce4e2a8,
nam=0xfffffe0017c43740,
td=0xfffffe00ae986000) at /usr/src/sys/netinet/udp_usrreq.c:1530
#12 0xffffffff80680615 in kern_connectat (td=0xfffffe00ae986000, dirfd=-100,
fd=<value optimized out>, sa=0xfffffe0017c43740) at
/usr/src/sys/kern/uipc_syscalls.c:596
#13 0xffffffff80680731 in sys_connect (td=0xfffffe00ae986000,
uap=0xffffff81aad4eb70)
at /usr/src/sys/kern/uipc_syscalls.c:562
#14 0xffffffff808e32cb in amd64_syscall (td=0xfffffe00ae986000, traced=0)
at subr_syscall.c:134
#15 0xffffffff808cde97 in Xfast_syscall () at
/usr/src/sys/amd64/amd64/exception.S:387
#16 0x0000000801b3d8da in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb) frame 6
#6 0xffffffff80736cec in in_pcblookup_local (pcbinfo=0xffffffff80dc9180, laddr=
{s_addr = 50374848}, lport=339, lookupflags=1, cred=0xfffffe016cdad100)
at /usr/src/sys/netinet/in_pcb.c:1438
1438 LIST_FOREACH(phd, porthash, phd_hash) {
(kgdb) list *0xffffffff80736cec
0xffffffff80736cec is in in_pcblookup_local
(/usr/src/sys/netinet/in_pcb.c:1439).
1434 * port hash list.
1435 */
1436 porthash =
&pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
1437 pcbinfo->ipi_porthashmask)];
1438 LIST_FOREACH(phd, porthash, phd_hash) {
1439 if (phd->phd_port == lport)
1440 break;
1441 }
1442 if (phd != NULL) {
1443 /*
(kgdb) p *pcbinfo
$1 = {ipi_lock = {lock_object = {lo_name = 0xffffffff809d4d82 "udp", lo_flags =
69926912,
lo_data = 0, lo_witness = 0x0}, rw_lock = 1}, ipi_listhead =
0xffffffff80dc9108,
ipi_count = 28, ipi_gencnt = 535501, ipi_lastport = 21249, ipi_lastlow = 0,
ipi_lasthi = 0, ipi_zone = 0xfffffe0017b60380, ipi_pcbgroups = 0x0,
ipi_npcbgroups = 0,
ipi_hashfields = 0, ipi_hash_lock = {lock_object = {
lo_name = 0xffffffff80a03d80 "pcbinfohash", lo_flags = 69402624, lo_data
= 0,
lo_witness = 0x0}, rw_lock = 18446741877615517696}, ipi_hashbase =
0xfffffe00120f6000,
ipi_hashmask = 127, ipi_porthashbase = 0xfffffe00120f5c04, ipi_porthashmask =
127,
ipi_wildbase = 0x0, ipi_wildmask = 0, ipi_vnet = 0x0, ipi_pspare = {0x0, 0x0}}
(kgdb) up
#7 0xffffffff807371d8 in in_pcb_lport (inp=0xfffffe016c2fb7a8,
laddrp=0xffffff81aad4e860,
lportp=0xffffff81aad4e86e, cred=0xfffffe016cdad100, lookupflags=1)
at /usr/src/sys/netinet/in_pcb.c:457
457 tmpinp = in_pcblookup_local(pcbinfo, laddr,
(kgdb) list *0xffffffff807371d8
0xffffffff807371d8 is in in_pcb_lport (/usr/src/sys/netinet/in_pcb.c:457).
452 #endif
453 #if defined(INET) && defined(INET6)
454 else
455 #endif
456 #ifdef INET
457 tmpinp = in_pcblookup_local(pcbinfo, laddr,
458 lport, lookupflags, cred);
459 #endif
460 } while (tmpinp != NULL);
461
(kgdb) up
#8 0xffffffff8073745a in in_pcbbind_setup (inp=0xfffffe016c2fb7a8, nam=0x0,
laddrp=0xffffff81aad4e900, lportp=0xffffff81aad4e90e,
cred=0xfffffe016cdad100)
at /usr/src/sys/netinet/in_pcb.c:615
615 error = in_pcb_lport(inp, &laddr, &lport, cred,
lookupflags);
(kgdb) list *0xffffffff8073745a
0xffffffff8073745a is in in_pcbbind_setup (/usr/src/sys/netinet/in_pcb.c:616).
611 }
612 if (*lportp != 0)
613 lport = *lportp;
614 if (lport == 0) {
615 error = in_pcb_lport(inp, &laddr, &lport, cred,
lookupflags);
616 if (error != 0)
617 return (error);
618
619 }
620 *laddrp = laddr.s_addr;
(kgdb) p nam
$2 = (struct sockaddr *) 0x0
(kgdb) root@orion:/usr/obj/usr/src/sys/ORION # ^D
Script done on Sat Apr 27 23:55:22 2013
pgpFx9YxZK939.pgp
Description: PGP signature
