--- Begin Message ---
Package: linux-image-2.6.24-etchnhalf.1-686
Version: 2.6.24-6~etchnhalf.5
Severity: normal
Hi,
I've written a web app that should be able to handle a lot of new
connections per second (1000+). On multiple servers I've hit a bug.
After running out of descriptors, then closing descriptors, epoll_wait
doesn't return anymore for the listen socket.
I've attached code to reproduce the issue. And an strace log. Even
before closing the descriptors you see epoll_wait already stops returning.
On the other side, I used a self-written app that just opens tons of
connections. Is there a standard utility to do that?
#include <arpa/inet.h>
#include <cassert>
#include <ctime>
#include <errno.h>
#include <netinet/in.h>
#include <sys/epoll.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <vector>
using namespace std;
int main()
{
int l = socket(AF_INET, SOCK_STREAM, 0);
unsigned long p = true;
ioctl(l, FIONBIO, &p);
sockaddr_in a = {0};
a.sin_family = AF_INET;
a.sin_addr.s_addr = INADDR_ANY;
a.sin_port = htons(2710);
bind(l, reinterpret_cast<sockaddr*>(&a), sizeof(sockaddr_in));
listen(l, SOMAXCONN);
int fd = epoll_create(1 << 10);
epoll_event e;
e.data.fd = l;
e.events = EPOLLIN | EPOLLOUT | EPOLLPRI | EPOLLERR | EPOLLHUP
| EPOLLET;
epoll_ctl(fd, EPOLL_CTL_ADD, l, &e);
const int c_events = 64;
epoll_event events[c_events];
typedef vector<int> sockets_t;
sockets_t sockets;
time_t t = time(NULL);
while (1)
{
int r = epoll_wait(fd, events, c_events, 5000);
if (r == -1)
continue;
if (!r && time(NULL) - t > 30)
{
for (int i = 0; i < sockets.size(); i++)
close(sockets[i]);
sockets.clear();
t = INT_MAX;
}
for (int i = 0; i < r; i++)
{
if (events[i].data.fd == l)
{
while (1)
{
int s = accept(l, NULL, NULL);
if (s == -1)
{
if (errno == EAGAIN)
break;
break; // continue;
}
sockets.push_back(s);
}
}
else
assert(false);
}
}
return 0;
}
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 3
ioctl(3, FIONBIO, [1]) = 0
bind(3, {sa_family=AF_INET, sin_port=htons(2710),
sin_addr=inet_addr("0.0.0.0")}, 16) = 0
listen(3, 128) = 0
epoll_create(1024) = 4
epoll_ctl(4, EPOLL_CTL_ADD, 3,
{EPOLLIN|EPOLLPRI|EPOLLOUT|EPOLLERR|EPOLLHUP|EPOLLET, {u32=3,
u64=13806959039201935363}}) = 0
time(NULL) = 1224527442
epoll_wait(4, {}, 64, 5000) = 0
time(NULL) = 1224527447
epoll_wait(4, {{EPOLLIN, {u32=3, u64=13806959039201935363}}}, 64, 5000) = 1
accept(3, 0, NULL) = 5
brk(0) = 0x804c000
brk(0x806d000) = 0x806d000
accept(3, 0, NULL) = 6
accept(3, 0, NULL) = 7
accept(3, 0, NULL) = 8
accept(3, 0, NULL) = -1 EAGAIN (Resource
temporarily unavailable)
epoll_wait(4, {{EPOLLIN, {u32=3, u64=13806959039201935363}}}, 64, 5000) = 1
accept(3, 0, NULL) = 9
...
accept(3, 0, NULL) = 85
accept(3, 0, NULL) = -1 EAGAIN (Resource
temporarily unavailable)
epoll_wait(4, {{EPOLLIN, {u32=3, u64=13806959039201935363}}}, 64, 5000) = 1
accept(3, 0, NULL) = 86
...
accept(3, 0, NULL) = 1023
accept(3, 0, NULL) = -1 EMFILE (Too many open files)
epoll_wait(4, {{EPOLLIN, {u32=3, u64=13806959039201935363}}}, 64, 5000) = 1
accept(3, 0, NULL) = -1 EMFILE (Too many open files)
epoll_wait(4, {{EPOLLIN, {u32=3, u64=13806959039201935363}}}, 64, 5000) = 1
...
epoll_wait(4, {{EPOLLIN, {u32=3, u64=13806959039201935363}}}, 64, 5000) = 1
accept(3, 0, NULL) = -1 EMFILE (Too many open files)
epoll_wait(4, {}, 64, 5000) = 0
time(NULL) = 1224527454
epoll_wait(4, {}, 64, 5000) = 0
time(NULL) = 1224527459
epoll_wait(4, {}, 64, 5000) = 0
time(NULL) = 1224527464
epoll_wait(4, {}, 64, 5000) = 0
time(NULL) = 1224527469
epoll_wait(4, {}, 64, 5000) = 0
time(NULL) = 1224527474
close(5) = 0
...
close(1023) = 0
epoll_wait(4, {}, 64, 5000) = 0
time(NULL) = 1224527479
epoll_wait(4, {}, 64, 5000) = 0
time(NULL) = 1224527484
epoll_wait(4, {}, 64, 5000) = 0
time(NULL) = 1224527489
epoll_wait(4, {}, 64, 5000) = 0
time(NULL) = 1224527494
epoll_wait(4, {}, 64, 5000) = 0
time(NULL) = 1224527499
epoll_wait(4, {}, 64, 5000) = 0
time(NULL) = 1224527504
-- Package-specific info:
** Version:
Linux version 2.6.24-etchnhalf.1-686 (Debian 2.6.24-6~etchnhalf.5)
(da...@debian.org) (gcc version 4.1.2 20061115 (prerelease) (Debian
4.1.1-21)) #1 SMP Mon Sep 8 06:19:11 UTC 2008
** Command line:
root=/dev/sda1 ro
** Not tainted
** Kernel log:
io scheduler anticipatory registered
io scheduler deadline registered
io scheduler cfq registered (default)
Limiting direct PCI/PCI transfers.
Boot video device is 0000:00:0f.0
isapnp: Scanning for PnP cards...
Switched to high resolution mode on CPU 1
Switched to high resolution mode on CPU 0
isapnp: No Plug & Play device found
Serial: 8250/16550 driver $Revision: 1.90 $ 4 ports, IRQ sharing enabled
serial8250: ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A
serial8250: ttyS1 at I/O 0x2f8 (irq = 3) is a 16550A
00:09: ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A
00:0a: ttyS1 at I/O 0x2f8 (irq = 3) is a 16550A
RAMDISK driver initialized: 16 RAM disks of 8192K size 1024 blocksize
PNP: PS/2 Controller [PNP0303:KBC,PNP0f13:MOUS] at 0x60,0x64 irq 1,12
serio: i8042 KBD port at 0x60,0x64 irq 1
serio: i8042 AUX port at 0x60,0x64 irq 12
mice: PS/2 mouse device common for all mice
cpuidle: using governor ladder
cpuidle: using governor menu
TCP bic registered
NET: Registered protocol family 1
NET: Registered protocol family 17
Using IPI No-Shortcut mode
registered taskstats version 1
input: AT Translated Set 2 keyboard as /class/input/input0
Freeing unused kernel memory: 248k freed
ACPI: Processor [CPU0] (supports 8 throttling states)
ACPI: Processor [CPU1] (supports 8 throttling states)
ACPI Exception (processor_core-0816): AE_NOT_FOUND, Processor Device is
not present [20070126]
ACPI Exception (processor_core-0816): AE_NOT_FOUND, Processor Device is
not present [20070126]
SCSI subsystem initialized
Fusion MPT base driver 3.04.06
Copyright (c) 1999-2007 LSI Corporation
Fusion MPT SPI Host driver 3.04.06
ACPI: PCI Interrupt 0000:00:10.0[A] -> GSI 17 (level, low) -> IRQ 16
mptbase: ioc0: Initiating bringup
pcnet32.c:v1.34-NAPI 14.Aug.2007 tsbog...@alpha.franken.de
libata version 3.00 loaded.
ioc0: LSI53C1030 B0: Capabilities={Initiator}
scsi0 : ioc0: LSI53C1030 B0, FwRev=00000000h, Ports=1, MaxQ=128, IRQ=16
ACPI: PCI Interrupt 0000:02:00.0[A] -> GSI 18 (level, low) -> IRQ 17
pcnet32: PCnet/PCI II 79C970A at 0x2000, 00 0c 29 5a 36 ae assigned IRQ 17.
eth0: registered as PCnet/PCI II 79C970A
pcnet32: 1 cards_found.
scsi 0:0:0:0: Direct-Access VMware, VMware Virtual S 1.0 PQ: 0 ANSI: 2
target0:0:0: Beginning Domain Validation
target0:0:0: Domain Validation skipping write tests
target0:0:0: Ending Domain Validation
target0:0:0: FAST-40 WIDE SCSI 80.0 MB/s ST (25 ns, offset 127)
Uniform Multi-Platform E-IDE driver Revision: 7.00alpha2
ide: Assuming 33MHz system bus speed for PIO modes; override with idebus=xx
PIIX4: IDE controller (0x8086:0x7111 rev 0x01) at PCI slot 0000:00:07.1
PCI: Enabling device 0000:00:07.1 (0000 -> 0001)
PIIX4: not 100% native mode: will probe irqs later
PIIX4: IDE port disabled
PIIX4: IDE port disabled
Driver 'sd' needs updating - please use bus_type methods
sd 0:0:0:0: [sda] 16777216 512-byte hardware sectors (8590 MB)
sd 0:0:0:0: [sda] Write Protect is off
sd 0:0:0:0: [sda] Mode Sense: 5d 00 00 00
sd 0:0:0:0: [sda] Cache data unavailable
sd 0:0:0:0: [sda] Assuming drive cache: write through
sd 0:0:0:0: [sda] 16777216 512-byte hardware sectors (8590 MB)
sd 0:0:0:0: [sda] Write Protect is off
sd 0:0:0:0: [sda] Mode Sense: 5d 00 00 00
sd 0:0:0:0: [sda] Cache data unavailable
sd 0:0:0:0: [sda] Assuming drive cache: write through
sda: sda1 sda2 < sda5 >
sd 0:0:0:0: [sda] Attached SCSI disk
Attempting manual resume
kjournald starting. Commit interval 5 seconds
EXT3-fs: mounted filesystem with ordered data mode.
Floppy drive(s): fd0 is 1.44M
FDC 0 is a post-1991 82077
input: PC Speaker as /class/input/input1
Real Time Clock Driver v1.12ac
pci_hotplug: PCI Hot Plug PCI Core version: 0.5
shpchp: Standard Hot Plug PCI Controller Driver version: 0.4
input: Power Button (FF) as /class/input/input2
Linux agpgart interface v0.102
agpgart: Detected an Intel 440BX Chipset.
agpgart: AGP aperture is 256M @ 0x0
parport_pc 00:08: reported by Plug and Play ACPI
parport0: PC-style at 0x378, irq 7 [PCSPP,TRISTATE]
input: ImPS/2 Generic Wheel Mouse as /class/input/input3
piix4_smbus 0000:00:07.3: Found 0000:00:07.3 device
piix4_smbus 0000:00:07.3: Host SMBus controller not enabled!
ACPI: Power Button (FF) [PWRF]
ACPI: AC Adapter [ACAD] (on-line)
Adding 409616k swap on /dev/sda5. Priority:-1 extents:1 across:409616k
EXT3 FS on sda1, internal journal
loop: module loaded
device-mapper: uevent: version 1.0.3
device-mapper: ioctl: 4.12.0-ioctl (2007-10-02) initialised:
dm-de...@redhat.com
eth0: link up
NET: Registered protocol family 10
lo: Disabled Privacy Extensions
eth0: no IPv6 routers present
** Loaded modules:
Module Size Used by
ipv6 242212 22
battery 13700 0
dm_snapshot 17060 0
dm_mirror 21792 0
dm_mod 56004 2 dm_snapshot,dm_mirror
loop 16996 0
i2c_piix4 8556 0
psmouse 36656 0
container 4960 0
parport_pc 26340 0
intel_agp 23540 1
agpgart 31912 1 intel_agp
ac 6212 0
i2c_core 22656 1 i2c_piix4
button 8528 0
shpchp 31220 0
pci_hotplug 27840 1 shpchp
parport 34472 1 parport_pc
serio_raw 6788 0
evdev 11200 0
rtc 13148 0
pcspkr 3264 0
floppy 54724 0
ext3 123272 1
jbd 43892 1 ext3
mbcache 8480 1 ext3
sd_mod 27296 3
generic 4484 0 [permanent]
piix 7556 0 [permanent]
ide_core 108740 2 generic,piix
ata_generic 7556 0
pcnet32 32196 0
mii 5344 1 pcnet32
libata 144560 1 ata_generic
mptspi 17992 2
mptscsih 25344 1 mptspi
mptbase 55172 2 mptspi,mptscsih
scsi_transport_spi 22976 1 mptspi
scsi_mod 141516 5
sd_mod,libata,mptspi,mptscsih,scsi_transport_spi
thermal 16092 0
processor 36840 1 thermal
fan 4868 0
** PCI devices:
not available
-- System Information:
Debian Release: 4.0
APT prefers stable
APT policy: (500, 'stable')
Architecture: i386 (i686)
Shell: /bin/sh linked to /bin/bash
Kernel: Linux 2.6.24-etchnhalf.1-686
Locale: LANG=en_US.UTF-8, LC_CTYPE=en_US.UTF-8 (charmap=UTF-8)
Versions of packages linux-image-2.6.24-etchnhalf.1-686 depends on:
ii debconf [debconf-2.0] 1.5.11etch2 Debian configuration
management sy
ii initramfs-tools [linux-initr 0.85i tools for generating an
initramfs
ii module-init-tools 3.3-pre4-2 tools for managing Linux
kernel mo
Versions of packages linux-image-2.6.24-etchnhalf.1-686 recommends:
ii libc6-i686 2.3.6.ds1-13etch7 GNU C Library: Shared
libraries [i
-- debconf information:
linux-image-2.6.24-etchnhalf.1-686/preinst/bootloader-initrd-2.6.24-etchnhalf.1-686:
true
linux-image-2.6.24-etchnhalf.1-686/preinst/elilo-initrd-2.6.24-etchnhalf.1-686:
true
shared/kernel-image/really-run-bootloader: true
linux-image-2.6.24-etchnhalf.1-686/preinst/abort-overwrite-2.6.24-etchnhalf.1-686:
linux-image-2.6.24-etchnhalf.1-686/postinst/depmod-error-2.6.24-etchnhalf.1-686:
false
linux-image-2.6.24-etchnhalf.1-686/postinst/old-dir-initrd-link-2.6.24-etchnhalf.1-686:
true
linux-image-2.6.24-etchnhalf.1-686/prerm/would-invalidate-boot-loader-2.6.24-etchnhalf.1-686:
true
linux-image-2.6.24-etchnhalf.1-686/preinst/overwriting-modules-2.6.24-etchnhalf.1-686:
true
linux-image-2.6.24-etchnhalf.1-686/preinst/initrd-2.6.24-etchnhalf.1-686:
linux-image-2.6.24-etchnhalf.1-686/postinst/old-system-map-link-2.6.24-etchnhalf.1-686:
true
linux-image-2.6.24-etchnhalf.1-686/postinst/depmod-error-initrd-2.6.24-etchnhalf.1-686:
false
linux-image-2.6.24-etchnhalf.1-686/preinst/lilo-initrd-2.6.24-etchnhalf.1-686:
true
linux-image-2.6.24-etchnhalf.1-686/postinst/kimage-is-a-directory:
linux-image-2.6.24-etchnhalf.1-686/preinst/abort-install-2.6.24-etchnhalf.1-686:
linux-image-2.6.24-etchnhalf.1-686/prerm/removing-running-kernel-2.6.24-etchnhalf.1-686:
true
linux-image-2.6.24-etchnhalf.1-686/preinst/failed-to-move-modules-2.6.24-etchnhalf.1-686:
linux-image-2.6.24-etchnhalf.1-686/preinst/lilo-has-ramdisk:
linux-image-2.6.24-etchnhalf.1-686/postinst/bootloader-error-2.6.24-etchnhalf.1-686:
linux-image-2.6.24-etchnhalf.1-686/postinst/bootloader-test-error-2.6.24-etchnhalf.1-686:
linux-image-2.6.24-etchnhalf.1-686/postinst/old-initrd-link-2.6.24-etchnhalf.1-686:
true
linux-image-2.6.24-etchnhalf.1-686/postinst/create-kimage-link-2.6.24-etchnhalf.1-686:
true
--- End Message ---
