All httpd child processes Segmentation fault near MaxRequestsPerChild, seemingly when it is time for the process to recycle. MaxRequestsPerChild currently set to 10000. I am also not able to reproduce this on a test system. Child process exits clean when attached with gdb. Child process also exits clean on stops/restarts.
[notice] child pid 14054 exit signal Segmentation fault (11) #Info: RHEL 6.4 Server version: Apache/2.2.15 (Unix) > Server built: Apr 29 2013 04:13:12 > Server's Module Magic Number: 20051115:25 > Server loaded: APR 1.3.9, APR-Util 1.3.9 > Compiled using: APR 1.3.9, APR-Util 1.3.9 > Architecture: 64-bit > Server MPM: Prefork > threaded: no > forked: yes (variable process count) > Server compiled with.... > -D APACHE_MPM_DIR="server/mpm/prefork" > -D APR_HAS_SENDFILE > -D APR_HAS_MMAP > -D APR_HAVE_IPV6 (IPv4-mapped addresses enabled) > -D APR_USE_SYSVSEM_SERIALIZE > -D APR_USE_PTHREAD_SERIALIZE > -D SINGLE_LISTEN_UNSERIALIZED_ACCEPT > -D APR_HAS_OTHER_CHILD > -D AP_HAVE_RELIABLE_PIPED_LOGS > -D DYNAMIC_MODULE_LIMIT=128 > -D HTTPD_ROOT="/etc/httpd" > -D SUEXEC_BIN="/usr/sbin/suexec" > -D DEFAULT_PIDLOG="run/httpd.pid" > -D DEFAULT_SCOREBOARD="logs/apache_runtime_status" > -D DEFAULT_LOCKFILE="logs/accept.lock" > -D DEFAULT_ERRORLOG="logs/error_log" > -D AP_TYPES_CONFIG_FILE="conf/mime.types" > -D SERVER_CONFIG_FILE="conf/httpd.conf" # [debug] prefork.c(1018): AcceptMutex: sysvsem (default: sysvsem) #gdb bt sample: Program terminated with signal 11, Segmentation fault. > #0 __pthread_mutex_lock (mutex=0x0) at pthread_mutex_lock.c:50 > 50 unsigned int type = PTHREAD_MUTEX_TYPE (mutex); #strace sample: *Filtered open("/etc/pki/nssdb/pkcs11.txt", O_RDONLY) = 23 > > close(23) = 0 > open("/etc/ld.so.cache", O_RDONLY) = 23 > close(23) = 0 > open("/usr/lib64/libnsssysinit.so", O_RDONLY) = 23 > close(23) = 0 > ... > open("/etc/pki/nssdb/cert9.db", O_RDONLY) = 23 > fcntl(23, F_GETFD) = 0 > fcntl(23, F_SETFD, FD_CLOEXEC) = 0 > fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, > st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0, > st_blksize=4096, st_blocks=24, st_size=9216, > st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17 > , st_ctime=2013/08/20-09:59:34}) = 0 > lseek(23, 0, SEEK_SET) = 0 > read(23, "SQLite format 3\0\4\0\1\1\0@ \0\0\0\1\0\0\0\0"..., > 100) = 100 > fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, > start=1073741824, len=1}) = 0 > fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, > start=1073741826, len=510}) = 0 > fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, > start=1073741824, len=1}) = 0 > fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, > st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0, > st_blksize=4096, st_blocks=24, st_size=9216, > st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17 > , st_ctime=2013/08/20-09:59:34}) = 0 > lseek(23, 0, SEEK_SET) = 0 > read(23, "SQLite format 3\0\4\0\1\1\0@ \0\0\0\1\0\0\0\0"..., > 1024) = 1024 > lseek(23, 3072, SEEK_SET) = 3072 > read(23, > "\r\0\0\0\1\0J\0\0J\3\315\0\0\0\0\0\0\0\0\0\0\0\2073\1\7\27\37\37\1\216"..., > 1024) = 1024 > lseek(23, 5120, SEEK_SET) = 5120 > read(23, > "\r\0\0\0\5\2\302\0\3\212\3\275\3D\3\4\2\302\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., > 1024) = 1024 > fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, > len=0}) = 0 > fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, > start=1073741824, len=1}) = 0 > fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, > start=1073741826, len=510}) = 0 > fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, > start=1073741824, len=1}) = 0 > fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, > st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0, > st_blksize=4096, st_blocks=24, st_size=9216, > st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17 > , st_ctime=2013/08/20-09:59:34}) = 0 > > ... > > rt_sigprocmask(SIG_BLOCK, [INT], NULL, 8) = 0 > rt_sigaction(SIGINT, {0x7f8dc4ec0e50, [], SA_RESTORER, > 0x7f8dc33f0920}, {0x7f8db3507f0c, ~[ILL ABRT BUS FPE KILL SEGV USR2 STOP > XCPU XFSZ SYS RTMIN RT_1], SA_RESTORER|SA_SIGINFO, 0x7f8dc33f0920}, 8) = 0 > rt_sigprocmask(SIG_UNBLOCK, [INT], NULL, 8) = 0 > munmap(0x7f8dae20f000, 2178616) = 0 > munmap(0x7f8daeced000, 2129488) = 0 > munmap(0x7f8daead9000, 2176864) = 0 > munmap(0x7f8dae89c000, 2343352) = 0 > munmap(0x7f8dae627000, 2574912) = 0 > munmap(0x7f8dae423000, 2109528) = 0 > munmap(0x7f8daeef5000, 2142192) = 0 > munmap(0x7f8daf100000, 2125616) = 0 > munmap(0x7f8daf307000, 2128512) = 0 > munmap(0x7f8daf50f000, 2107408) = 0 > munmap(0x7f8daf712000, 2103968) = 0 > munmap(0x7f8daf914000, 2111904) = 0 > munmap(0x7f8dafb18000, 2162656) = 0 > munmap(0x7f8dafd28000, 2140264) = 0 > munmap(0x7f8db0911000, 2116672) = 0 > munmap(0x7f8db044f000, 2099376) = 0 > munmap(0x7f8db0650000, 2885912) = 0 > munmap(0x7f8db0b16000, 2123736) = 0 > munmap(0x7f8db0d1d000, 2357704) = 0 > munmap(0x7f8db11ec000, 2119816) = 0 > munmap(0x7f8db13f2000, 2125296) = 0 > munmap(0x7f8db15f9000, 2198440) = 0 > munmap(0x7f8db467b000, 2289928) = 0 > munmap(0x7f8db48ab000, 2232272) = 0 > munmap(0x7f8db4e50000, 2149768) = 0 > munmap(0x7f8db4acc000, 3684744) = 0 > munmap(0x7f8db5290000, 2142072) = 0 > munmap(0x7f8db505d000, 2303840) = 0 > munmap(0x7f8db549b000, 4278992) = 0 > munmap(0x7f8db58b0000, 2148288) = 0 > munmap(0x7f8db5abd000, 2131392) = 0 > munmap(0x7f8db6d48000, 2440032) = 0 > munmap(0x7f8db6aab000, 2737840) = 0 > munmap(0x7f8db655d000, 2165496) = 0 > munmap(0x7f8db676e000, 3394936) = 0 > munmap(0x7f8db6337000, 2250552) = 0 > munmap(0x7f8db60e7000, 2423648) = 0 > munmap(0x7f8db5ec9000, 2217576) = 0 > munmap(0x7f8db5cc6000, 2106112) = 0 > munmap(0x7f8db6f9c000, 3939304) = 0 > munmap(0x7f8db735e000, 2268048) = 0 > fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741824, > len=1}) = 0 > fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741826, > len=510}) = 0 > fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=1073741824, > len=1}) = 0 > access("/etc/pki/nssdb/cert9.db-journal", F_OK) > = -1 ENOENT (No such file or directory) > fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, st_mode=S_IFREG|0644, > st_nlink=1, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=24, > st_size=9216, st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17, > st_ctime=2013/08/20-09:59:34}) = 0 > lseek(23, 24, SEEK_SET) = 24 > read(23, "\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\0", 16) = 16 > fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, len=0}) = 0 > fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741824, > len=1}) = 0 > fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741826, > len=510}) = 0 > fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=1073741824, > len=1}) = 0 > access("/etc/pki/nssdb/cert9.db-journal", F_OK) = -1 ENOENT (No such > file or directory) > fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, st_mode=S_IFREG|0644, > st_nlink=1, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=24, > st_size=9216, st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17, > st_ctime=2013/08/20-09:59:34}) = 0 > lseek(23, 24, SEEK_SET) = 24 > read(23, "\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\0", 16) = 16 > fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, len=0}) = 0 > brk(0x7f8dc6191000) = 0x7f8dc6191000 > munmap(0x7f8db7e3e000, 2161288) = 0 > munmap(0x7f8db7bea000, 2438600) = 0 > munmap(0x7f8db79b8000, 2301736) = 0 > munmap(0x7f8db7588000, 2257152) = 0 > close(8) = 0 > close(7) = 0 > fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741824, > len=1}) = 0 > fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741826, > len=510}) = 0 > fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=1073741824, > len=1}) = 0 > access("/etc/pki/nssdb/cert9.db-journal", F_OK) = -1 ENOENT (No such > file or directory) > fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, st_mode=S_IFREG|0644, > st_nlink=1, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=24, > st_size=9216, st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17, > st_ctime=2013/08/20-09:59:34}) = 0 > lseek(23, 24, SEEK_SET) = 24 > read(23, "\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\0", 16) = 16 > fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, len=0}) = 0 > --- SIGSEGV (Segmentation fault) @ 0 (0) --- > chdir("/apache2/apache-dumps") = 0 > rt_sigaction(SIGSEGV, {SIG_DFL, [], SA_RESTORER|SA_INTERRUPT, > 0x7f8dc3760500}, {SIG_DFL, [], SA_RESTORER|SA_RESETHAND, 0x7f8dc3760500}, > 8) = 0 > kill(26317, SIGSEGV) = 0 > rt_sigreturn(0x66cd) = 140246846382624 > --- SIGSEGV (Segmentation fault) @ 0 (0) --- > >