hello,
I have a setup with two carp nodes, I was replacing one of these with
new hardware (old was i386, new is amd64) and installed latest snapshot
and then copied over all the relevant files from the old node, changing
things which differed (vr -> em). I shut the old host down, and booted
the new host. Once it gets to starting network it panics. ddb led me to
believe it was vlan related, so I moved all hostname.* files out of the
way which were not related to the physical interfaces. At that point the
machine boots.
since the trace showed some arp things in the ddb trace I thought that
perhaps something was being cached somewhere, so on a whim I tried
incrementing the IP on one of the vlan interfaces by one in single user
mode and then doing sh /etc/netstart vlan666 all of a sudden it worked.
Thinking I'd solved it I did the same for all vlan interfaces and exited
single user mode, and it paniced. I tried a few more times, swapping IPs
or starting a vlan at a time, sometimes I'd get one up, once I even got
all vlans up in single user mode but when going multiuser it would
panic. My last attempt now even in single user mode it panics for each
vlan I try.
ddb info for two panics, ifconfig and hostname* files as well as a dmesg
below.
thanks,
.jh
First panic (during multiuser boot)
starting network
uvm_fault(0xffffff007f6d13c0, 0x8, 0, 2) -> e
kernel: page fault trap, code=0
Stopped at if_ref+0x8: lock incl 0x8(%rdi)
ddb{1}> ps
PID PPID PGRP UID S FLAGS WAIT COMMAND
*28769 12596 31290 0 7 0x3 ifconfig
12596 31290 31290 0 3 0x8b pause sh
31290 1 31290 0 3 0x8b pause sh
19920 0 0 0 3 0x14200 pgzero zerothread
25482 0 0 0 3 0x14200 aiodoned aiodoned
13982 0 0 0 3 0x14200 syncer update
17550 0 0 0 3 0x14200 cleaner cleaner
31713 0 0 0 3 0x14200 reaper reaper
31674 0 0 0 3 0x14200 pgdaemon pagedaemon
2400 0 0 0 3 0x14200 bored crypto
23244 0 0 0 3 0x14200 pftm pfpurge
15871 0 0 0 3 0x14200 mmctsk sdmmc1
3329 0 0 0 3 0x14200 mmctsk sdmmc0
21406 0 0 0 3 0x14200 usbtsk usbtask
15224 0 0 0 3 0x14200 usbatsk usbatsk
6868 0 0 0 3 0x40014200 idle1
17764 0 0 0 3 0x14200 bored sensors
29990 0 0 0 3 0x14200 bored softnet
3936 0 0 0 3 0x14200 bored systqmp
26693 0 0 0 3 0x14200 bored systq
11761 0 0 0 7 0x40014200 idle0
1 0 1 0 3 0x82 wait init
0 -1 0 0 3 0x10200 scheduler swapper
ddb{1}> trace
if_ref() at if_ref+0x8
vlan_start() at vlan_start+0x107
if_enqueue() at if_enqueue+0x11a
ether_output() at ether_output+0x16e
arprequest() at arprequest+0x11d
arp_rtrequest() at arp_rtrequest+0x154
rtrequest1() at rtrequest1+0x656
rt_ifa_add() at rt_ifa_add+0x10b
in_ifinit() at in_ifinit+0x101
in_control() at in_control+0x457
ifioctl() at ifioctl+0x175
sys_ioctl() at sys_ioctl+0x18b
syscall() at syscall+0x358
--- syscall (number 54) ---
end of kernel
end trace frame: 0x136f3db4f5c0, count: -13
0x136f3d915d6a:
ddb{1}> show registers
rdi 0
rsi 0x7
rbp 0xffff80002125e820
rbx 0xffffff006f2c6c00
rdx 0
rcx 0xffffff006f2c6c00
rax 0
r8 0x608
r9 0x1
r10 0x1
r11 0xffffffff81211110 vlan_start
r12 0xffff80000012e800
r13 0x3
r14 0xffff800000073048
r15 0xffff80000012e800
rip 0xffffffff811f3978 if_ref+0x8
cs 0x8
rflags 0x10282 __ALIGN_SIZE+0xf282
rsp 0xffff80002125e810
ss 0x10
if_ref+0x8: lock incl 0x8(%rdi)
A later panic while trying sh /etc/netstart vlan20 in single user mode
# sh /etc/netstart vlan20
uvm_fault(0xffffff007f6d12d0, 0x8, 0, 2) -> e
kernel: page fault trap, code=0
Stopped at if_ref+0x8: lock incl 0x8(%rdi)
ddb{0}> trace
if_ref() at if_ref+0x8
vlan_start() at vlan_start+0x107
if_enqueue() at if_enqueue+0x11a
ether_output() at ether_output+0x16e
arprequest() at arprequest+0x11d
arp_rtrequest() at arp_rtrequest+0x154
rtrequest1() at rtrequest1+0x656
rt_ifa_add() at rt_ifa_add+0x10b
in_ifinit() at in_ifinit+0x101
in_control() at in_control+0x457
ifioctl() at ifioctl+0x175
sys_ioctl() at sys_ioctl+0x18b
syscall() at syscall+0x358
--- syscall (number 54) ---
end of kernel
end trace frame: 0x120ad4f5c0, count: -13
0x120ab15d6a:
ddb{0}> ps
PID PPID PGRP UID S FLAGS WAIT COMMAND
* 6728 11984 11984 0 7 0x3 ifconfig
11984 11648 11984 0 3 0x8b pause sh
11648 1 11648 0 3 0x8b pause sh
31080 0 0 0 3 0x14200 pgzero zerothread
6836 0 0 0 3 0x14200 aiodoned aiodoned
8714 0 0 0 3 0x14200 syncer update
10213 0 0 0 3 0x14200 cleaner cleaner
31169 0 0 0 3 0x14200 reaper reaper
22991 0 0 0 3 0x14200 pgdaemon pagedaemon
20703 0 0 0 3 0x14200 bored crypto
29436 0 0 0 3 0x14200 pftm pfpurge
32279 0 0 0 3 0x14200 mmctsk sdmmc1
17413 0 0 0 3 0x14200 mmctsk sdmmc0
13439 0 0 0 3 0x14200 usbtsk usbtask
24962 0 0 0 3 0x14200 usbatsk usbatsk
11964 0 0 0 7 0x40014200 idle1
25255 0 0 0 3 0x14200 bored sensors
25938 0 0 0 3 0x14200 bored softnet
11751 0 0 0 3 0x14200 bored systqmp
14554 0 0 0 3 0x14200 bored systq
15780 0 0 0 3 0x40014200 idle0
1 0 1 0 3 0x82 wait init
0 -1 0 0 3 0x10200 scheduler swapper
ddb{0}> show registers
rdi 0
rsi 0x7
rbp 0xffff80002125e820
rbx 0xffffff006f6dcc00
rdx 0
rcx 0xffffff006f6dcc00
rax 0
r8 0x608
r9 0x1
r10 0x1
r11 0xffffffff81211110 vlan_start
r12 0xffff8000000d8000
r13 0x3
r14 0xffff800000073048
r15 0xffff8000000d8000
rip 0xffffffff811f3978 if_ref+0x8
cs 0x8
rflags 0x10282 __ALIGN_SIZE+0xf282
rsp 0xffff80002125e810
ss 0x10
if_ref+0x8: lock incl 0x8(%rdi)
ddb{0}> mach ddbcpu 1
Stopped at Debugger+0x9: leave
ddb{1}> trace
Debugger() at Debugger+0x9
x86_ipi_handler() at x86_ipi_handler+0x76
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x1c
--- interrupt ---
cpu_idle_mwait_cycle() at cpu_idle_mwait_cycle+0x62
cpu_idle_cycle() at cpu_idle_cycle+0x10
end trace frame: 0x0, count: -5
# ifconfig
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> mtu 32768
priority: 0
groups: lo
inet6 ::1 prefixlen 128
inet6 fe80::1%lo0 prefixlen 64 scopeid 0x6
inet 127.0.0.1 netmask 0xff000000
em0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:00:24:d1:ad:60
priority: 0
groups: egress
media: Ethernet autoselect (1000baseT full-duplex,master)
status: active
inet 172.16.0.3 netmask 0xffffff00 broadcast 172.16.0.255
em1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:00:24:d1:ad:61
priority: 0
media: Ethernet autoselect (1000baseT full-duplex)
status: active
em2: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:00:24:d1:ad:62
priority: 0
media: Ethernet autoselect (1000baseT full-duplex)
status: active
em3: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr 00:00:24:d1:ad:63
priority: 0
media: Ethernet autoselect (1000baseT full-duplex,master)
status: active
enc0: flags=0<>
priority: 0
groups: enc
status: active
tun0: flags=8051<UP,POINTOPOINT,RUNNING,MULTICAST> mtu 1500
priority: 0
groups: tun
status: active
inet 10.6.6.1 --> 10.6.6.2 netmask 0xffffffff
pflog0: flags=141<UP,RUNNING,PROMISC> mtu 33144
priority: 0
groups: pflog
Here are all the hostname.* files not displayed above
# cat hostname.carp0
inet 172.16.0.103 255.255.255.0 172.16.0.255 vhid 1 carpdev em0 pass X
carppeer 172.16.0.2
# cat hostname.carp1
inet 192.168.100.103 255.255.255.0 192.168.100.255 vhid 2 carpdev vlan20
pass XX carppeer 192.168.100.2
# cat hostname.carp2
inet 192.168.0.103 255.255.255.0 192.168.0.255 vhid 3 carpdev vlan30
pass XXX carppeer 192.168.0.2
# cat hostname.carp3
inet 10.66.66.103 255.255.255.0 10.66.66.255 vhid 4 carpdev vlan666 pass
XXXX carppeer 10.66.66.2
# cat hostname.pfsync0
up syncdev vlan666 syncpeer 10.66.66.2
# cat hostname.vlan20
inet 192.168.100.3 255.255.255.0 NONE vlan 20 vlandev em1
# cat hostname.vlan30
inet 192.168.0.3 255.255.255.0 NONE vlan 30 vlandev em2
# cat hostname.vlan666
inet 10.66.66.3 255.255.255.0 NONE vlan 666 vlandev em3
dmesg:
OpenBSD 5.8-current (GENERIC.MP) #1364: Wed Sep 9 17:32:01 MDT 2015
dera...@amd64.openbsd.org:/usr/src/sys/arch/amd64/compile/GENERIC.MP
real mem = 2130575360 (2031MB)
avail mem = 2062110720 (1966MB)
mpath0 at root
scsibus0 at mpath0: 256 targets
mainbus0 at root
bios0 at mainbus0
acpi at bios0 not configured
mpbios0 at bios0: Intel MP Specification 1.4
cpu0 at mainbus0: apid 0 (boot processor)
cpu0: Genuine Intel(R) CPU @ 1.60GHz, 1600.22 MHz
cpu0:
FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,CFLUSH,DS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM,PBE,SSE3,DTES64,MWAIT,DS-CPL,VMX,EST,TM2,SSSE3,CX16,xTPR,PDCM,MOVBE,NXE,LONG,LAHF,PERF,SENSOR
cpu0: 512KB 64b/line 8-way L2 cache
cpu0: smt 0, core 0, package 0
mtrr: Pentium Pro MTRR support, 8 var ranges, 88 fixed ranges
cpu0: apic clock running at 99MHz
cpu0: mwait min=64, max=64, C-substates=0.2.2.0.2.0.3, IBE
cpu1 at mainbus0: apid 1 (application processor)
cpu1: Genuine Intel(R) CPU @ 1.60GHz, 1599.99 MHz
cpu1:
FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,CFLUSH,DS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM,PBE,SSE3,DTES64,MWAIT,DS-CPL,VMX,EST,TM2,SSSE3,CX16,xTPR,PDCM,MOVBE,NXE,LONG,LAHF,PERF,SENSOR
cpu1: 512KB 64b/line 8-way L2 cache
cpu1: smt 1, core 0, package 0
mpbios0: bus 0 is type PCI
mpbios0: bus 64 is type ISA
ioapic0 at mainbus0: apid 0 pa 0xfec00000, version 20, 24 pins
cpu0: unknown Enhanced SpeedStep CPU, msr 0x0612101906001019
cpu0: using only highest and lowest power states
cpu0: Enhanced SpeedStep 1600 MHz: speeds: 1600, 600 MHz
pci0 at mainbus0 bus 0
pchb0 at pci0 dev 0 function 0 "Intel E600 Host" rev 0x05
pchb1 at pci0 dev 1 function 0 "Intel E600 Config" rev 0x00
ppb0 at pci0 dev 23 function 0 "Intel E600 PCIE" rev 0x00
pci1 at ppb0 bus 1
ppb1 at pci1 dev 0 function 0 "Intel EG20T PCIE" rev 0x01
pci2 at ppb1 bus 2
"Intel EG20T Packet Hub" rev 0x01 at pci2 dev 0 function 0 not configured
"Intel EG20T Ethernet" rev 0x02 at pci2 dev 0 function 1 not configured
"Intel EG20T GPIO" rev 0x01 at pci2 dev 0 function 2 not configured
ohci0 at pci2 dev 2 function 0 "Intel EG20T USB" rev 0x02: apic 0 int
19, version 1.0
ohci1 at pci2 dev 2 function 1 "Intel EG20T USB" rev 0x02: apic 0 int
19, version 1.0
ohci2 at pci2 dev 2 function 2 "Intel EG20T USB" rev 0x02: apic 0 int
19, version 1.0
ehci0 at pci2 dev 2 function 3 "Intel EG20T USB" rev 0x02: apic 0 int 19
usb0 at ehci0: USB revision 2.0
uhub0 at usb0 "Intel EHCI root hub" rev 2.00/1.00 addr 1
"Intel EG20T USB Client" rev 0x02 at pci2 dev 2 function 4 not configured
sdhc0 at pci2 dev 4 function 0 "Intel EG20T SDIO" rev 0x01: apic 0 int 18
sdmmc0 at sdhc0
sdhc1 at pci2 dev 4 function 1 "Intel EG20T SDIO" rev 0x01: apic 0 int 18
sdmmc1 at sdhc1
ahci0 at pci2 dev 6 function 0 "Intel EG20T AHCI" rev 0x02: msi, AHCI 1.1
ahci0: port 0: 3.0Gb/s
scsibus1 at ahci0: 32 targets
sd0 at scsibus1 targ 0 lun 0: <ATA, KingSpec KSM-mSA, 1110> SCSI3
0/direct fixed t10.ATA_KingSpec_KSM-mSATA.5-016SJ_MSA0511122700019_
sd0: 15272MB, 512 bytes/sector, 31277232 sectors
ohci3 at pci2 dev 8 function 0 "Intel EG20T USB" rev 0x02: apic 0 int
16, version 1.0
ohci4 at pci2 dev 8 function 1 "Intel EG20T USB" rev 0x02: apic 0 int
16, version 1.0
ohci5 at pci2 dev 8 function 2 "Intel EG20T USB" rev 0x02: apic 0 int
16, version 1.0
ehci1 at pci2 dev 8 function 3 "Intel EG20T USB" rev 0x02: apic 0 int 16
usb1 at ehci1: USB revision 2.0
uhub1 at usb1 "Intel EHCI root hub" rev 2.00/1.00 addr 1
"Intel EG20T DMA" rev 0x00 at pci2 dev 10 function 0 not configured
puc0 at pci2 dev 10 function 1 "Intel EG20T Serial" rev 0x01: ports: 1 com
com4 at puc0 port 0 apic 0 int 19: ti16750, 64 byte fifo
puc1 at pci2 dev 10 function 2 "Intel EG20T Serial" rev 0x00: ports: 1 com
com5 at puc1 port 0 apic 0 int 19: ti16750, 64 byte fifo
puc2 at pci2 dev 10 function 3 "Intel EG20T Serial" rev 0x00: ports: 1 com
com6 at puc2 port 0 apic 0 int 19: ti16750, 64 byte fifo
puc3 at pci2 dev 10 function 4 "Intel EG20T Serial" rev 0x00: ports: 1 com
com7 at puc3 port 0 apic 0 int 19: ti16750, 64 byte fifo
"Intel EG20T DMA" rev 0x00 at pci2 dev 12 function 0 not configured
"Intel EG20T SPI" rev 0x00 at pci2 dev 12 function 1 not configured
"Intel EG20T I2C" rev 0x00 at pci2 dev 12 function 2 not configured
"Intel EG20T CAN" rev 0x00 at pci2 dev 12 function 3 not configured
"Intel EG20T 1588" rev 0x01 at pci2 dev 12 function 4 not configured
usb2 at ohci0: USB revision 1.0
uhub2 at usb2 "Intel OHCI root hub" rev 1.00/1.00 addr 1
usb3 at ohci1: USB revision 1.0
uhub3 at usb3 "Intel OHCI root hub" rev 1.00/1.00 addr 1
usb4 at ohci2: USB revision 1.0
uhub4 at usb4 "Intel OHCI root hub" rev 1.00/1.00 addr 1
usb5 at ohci3: USB revision 1.0
uhub5 at usb5 "Intel OHCI root hub" rev 1.00/1.00 addr 1
usb6 at ohci4: USB revision 1.0
uhub6 at usb6 "Intel OHCI root hub" rev 1.00/1.00 addr 1
usb7 at ohci5: USB revision 1.0
uhub7 at usb7 "Intel OHCI root hub" rev 1.00/1.00 addr 1
ppb2 at pci0 dev 24 function 0 "Intel E600 PCIE" rev 0x00
pci3 at ppb2 bus 3
ppb3 at pci3 dev 0 function 0 "IDT 89HPES4T4" rev 0x0e
pci4 at ppb3 bus 4
ppb4 at pci4 dev 2 function 0 "IDT 89HPES4T4" rev 0x0e
pci5 at ppb4 bus 5
em0 at pci5 dev 0 function 0 "Intel 82574L" rev 0x00: msi, address
00:00:24:d1:ad:60
ppb5 at pci4 dev 3 function 0 "IDT 89HPES4T4" rev 0x0e
pci6 at ppb5 bus 6
em1 at pci6 dev 0 function 0 "Intel 82574L" rev 0x00: msi, address
00:00:24:d1:ad:61
ppb6 at pci4 dev 4 function 0 "IDT 89HPES4T4" rev 0x0e
pci7 at ppb6 bus 7
ppb7 at pci0 dev 25 function 0 "Intel E600 PCIE" rev 0x00
pci8 at ppb7 bus 8
ppb8 at pci8 dev 0 function 0 "IDT 89HPES4T4" rev 0x0e
pci9 at ppb8 bus 9
ppb9 at pci9 dev 2 function 0 "IDT 89HPES4T4" rev 0x0e
pci10 at ppb9 bus 10
em2 at pci10 dev 0 function 0 "Intel 82574L" rev 0x00: msi, address
00:00:24:d1:ad:62
ppb10 at pci9 dev 3 function 0 "IDT 89HPES4T4" rev 0x0e
pci11 at ppb10 bus 11
em3 at pci11 dev 0 function 0 "Intel 82574L" rev 0x00: msi, address
00:00:24:d1:ad:63
ppb11 at pci9 dev 4 function 0 "IDT 89HPES4T4" rev 0x0e
pci12 at ppb11 bus 12
ppb12 at pci0 dev 26 function 0 "Intel E600 PCIE" rev 0x00
pci13 at ppb12 bus 13
tcpcib0 at pci0 dev 31 function 0 "Intel E600 LPC" rev 0x00: 14318179 Hz
timer, watchdog
isa0 at tcpcib0
isadma0 at isa0
com0 at isa0 port 0x3f8/8 irq 4: ti16750, 64 byte fifo
com0: console
pcppi0 at isa0 port 0x61
spkr0 at pcppi0
skgpio0 at isa0 port 0x680/32
gpio0 at skgpio0: 16 pins
gpio1 at skgpio0: 2 pins
vscsi0 at root
scsibus2 at vscsi0: 256 targets
softraid0 at root
scsibus3 at softraid0: 256 targets
root on sd0a (6fb9381c12ea5fc1.a) swap on sd0b dump on sd0b
