Ok,
Just happened again here's all the information that I have:-
netstat -LAan
Current listen queue sizes (qlen/incqlen/maxqlen)
Tcpcb Proto Listen Local Address
fffffe04c3ddeb70 tcp4 0/0/512 *.143
fffffe001ede47a0 tcp4 769/0/512 *.110 <----- max queue fills up (and
reaches backlog setting)
fffffe039c5e37a0 tcp46 0/0/459 *.3306
fffffe04ac4d6b70 tcp4 0/0/1024 127.0.0.1.2812
fffffe05a1b943d0 tcp4 0/0/128 *.28
fffffe00934827a0 tcp4 0/0/16 *.2000
fffffe00931873d0 tcp4 0/0/128 *.22
fffffe00931877a0 tcp6 0/0/128 *.22
fffffe00935317a0 tcp4 0/0/100 *.25
fffffe00933b97a0 tcp4 0/0/128 *.199
[root@redacted ~]# dmesg
sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue
awaiting acceptance
sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue
awaiting acceptance
sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue
awaiting acceptance
sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue
awaiting acceptance
sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue
awaiting acceptance
sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue
awaiting acceptance
sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue
awaiting acceptance
sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue
awaiting acceptance
sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue
awaiting acceptance
sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue
awaiting acceptance
sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue
awaiting acceptance
[root@ redacted ~]# telnet localhost 110
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
Connection closed by foreign host.
[root@redacted ]# top
last pid: 86783; load averages: 1.13, 1.10, 0.92
up
5+23:11:40 14:12:24
33 processes: 1 running, 32 sleeping
CPU: % user, % nice, % system, % interrupt, % idle
Mem: 12G Active, 1608M Inact, 8703M Wired, 33M Cache, 1046M Free
ARC: 7592M Total, 2723M MFU, 4327M MRU, 24M Anon, 166M Header, 363M Other
PID USERNAME THR PRI NICE SIZE RES STATE C TIME WCPU COMMAND
79266 nobody 2 20 0 168M 78248K uwait 0 86:59 100.00%
dbmail-pop3d <---- using 100% CPU
78725 mysql 38 20 0 14861M 13127M uwait 1 322:42 9.23% mysqld
86766 nobody 12 20 0 265M 95552K uwait 5 0:01 0.88%
dbmail-imapd
35214 nobody 1 20 0 128M 46364K kqread 2 59:31 0.39%
dbmail-lmtpd
1016 root 1 20 0 47756K 2480K select 3 63:03 0.10% snmpd
1047 root 1 20 0 12084K 888K select 6 169:26 0.00% powerd
950 root 1 20 0 12084K 1052K select 10 22:44 0.00% syslogd
5300 root 2 20 0 33116K 1728K select 9 4:36 0.00% monit
Trace output:-
rite(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource
temporarily unavailable'
procstat:-
PID TID COMM TDNAME CPU PRI STATE WCHAN
79266 101601 dbmail-pop3d - 1 203 run -
79266 102226 dbmail-pop3d - 0 120 sleep uwait
GDB output:-
[New Thread 80500a400 (LWP 102226)]
[New Thread 805007400 (LWP 101601)]
[Switching to Thread 80500a400 (LWP 102226)]
0x0000000803ed64dc in ?? () from /lib/libthr.so.3
[Switching to thread 2 (Thread 805007400 (LWP 101601))]
#0 0x00000008041f2afc in write () from /lib/libc.so.7
#0 0x00000008041f2afc in write () from /lib/libc.so.7
No symbol table info available.
#1 0x0000000803ed0ce0 in write () from /lib/libthr.so.3
No symbol table info available.
#2 0x0000000800864390 in ci_write (client=0x805d7e000, msg=<optimized out>) at
clientbase.c:318
ap = {{gp_offset = 1835430759, fp_offset = 2034895373,
overflow_arg_area = 0x786f5563366a6a44, reg_save_area = 0x47464b7845797964}}
cp = {{gp_offset = 4294942656, fp_offset = 32767, overflow_arg_area =
0x803ed57a4, reg_save_area = 0x193}}
t = <optimized out>
e = <optimized out>
n = 262143
left = 1096120
s = 0x8096261e4
"avpOnW/i/VtPD22raFLIXTzEKrGcn5B64I+U+hHevp61XofCwpI93+Gnhbw1rP\r\nxi17wvbeNZbLTvB2kziHWdPiivxKsFtJLuCv8sqgFs7uysMA18j/ALTniLVvGHwf+Ira1aad/Zf2\r\nWJleCH7KkzRgDeFySmSzEDgewAr0zRbSLR7XXNSjtLuxbTJFVJBCXVV3Oo"...
buf = "39jXtyUbpLh4B9cAD9K+jtPkijhKSY3zk7Bn\r\nn"
__func__ = "ci_write"
#3 0x000000080086446c in ci_write_cb (client=0x805d7e000) at clientbase.c:263
rest = <optimized out>
result = <optimized out>
#4 0x0000000800863095 in client_session_bailout (session=<optimized out>) at
clientsession.c:146
c = 0x805269f00
args = 0x0
from = 0x0
rcpt = 0x0
messagelst = 0x0
client = 0x805d7e000
__func__ = "client_session_bailout"
#5 0x0000000000402285 in pop3_cb_time (arg=<optimized out>) at pop3.c:272
session = 0x805269f00
#6 0x0000000802961aee in event_base_loop () from
/usr/local/lib/event2/libevent-2.0.so.6
No symbol table info available.
#7 0x00000008008628a1 in server_run (conf=0x7fffffffa310) at server.c:842
i = 1
__func__ = "server_run"
#8 0x0000000800862d6d in server_mainloop (config=0x7fffffffa310,
service=0x403b3a "POP", servicename=<optimized out>) at server.c:976
__func__ = "server_mainloop"
#9 0x0000000000403a37 in main (argc=1, argv=<optimized out>) at pop3d.c:52
config = {no_daemonize = 0, log_verbose = 0, pidFile = 0x805017d80
"/var/run/dbmail/dbmail-pop3d.pid", timeout = 120, login_timeout = 30, iplist =
0x8050dd920, port = "110", '\000' <repeats 1020 times>, ssl_port = '\000'
<repeats 1023 times>, ipcount = 1, socketcount = 1, ssl_socketcount = 0,
listenSockets = 0x80500a800, ssl_listenSockets = 0x80500ac00,
service_before_smtp = 0, authlog = 0, ssl = 0, backlog = 512, resolveIP = 0,
evh = 0x0, service_name = "POP", '\000' <repeats 1020 times>, process_name =
"dbmail-pop3d", '\000' <repeats 1011 times>, serverUser = "nobody", '\000'
<repeats 1017 times>, serverGroup = "nogroup", '\000' <repeats 1016 times>,
socket = '\000' <repeats 1023 times>, log = "/var/log/dbmail.log", '\000'
<repeats 1004 times>, error_log = "/var/log/dbmail.err", '\000' <repeats 1004
times>, pid_dir = "/var/run/dbmail", '\000' <repeats 1008 times>, tls_cafile =
'\000' <repeats 1023 times>, tls_cert = '\000' <repeats 1023 times>, tls_key =
'\000' <rep
eats 1023 times>, tls_ciphers = '\000' <repeats 1023 times>, ClientHandler =
0x402060 <pop3_handle_connection>, cb = 0x0}
result = 0
__func__ = "main"
Best regards,
James Greig
-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of
James Greig
Sent: 17 March 2014 10:54
To: DBMail mailinglist
Subject: Re: [Dbmail] Iphone HTML emails
Just had a short spike on pop3d but it recovered fairly quickly so probably
nothing significant here until it happens for a prolonged period again. The
main thing I tend to look at is the listen queue which I've set the backlog at
512 now
79266 nobody 2 20 0 164M 85344K uwait 5 46:06 100.00%
dbmail-pop3d
netstat -LAan
Tcpcb Proto Listen Local Address
fffffe0061598b70 tcp4 0/0/512 *.143
fffffe001ede47a0 tcp4 157/0/512 *.110 <---- pop3d listen queue jumbs up
to 157
fffffe039c5e37a0 tcp46 0/0/459 *.3306
fffffe04ac4d6b70 tcp4 0/0/1024 127.0.0.1.2812
fffffe05a1b943d0 tcp4 0/0/128 *.28
fffffe00934827a0 tcp4 0/0/16 *.2000
fffffe00931873d0 tcp4 0/0/128 *.22
fffffe00931877a0 tcp6 0/0/128 *.22
fffffe00935317a0 tcp4 0/0/100 *.25
fffffe00933b97a0 tcp4 0/0/128 *.199
PID TID COMM TDNAME CPU PRI STATE WCHAN
79266 101601 dbmail-pop3d - 3 203 run -
79266 102226 dbmail-pop3d - 5 120 sleep uwait
[New Thread 80500a400 (LWP 102226)]
[New Thread 805007400 (LWP 101601)]
[Switching to Thread 80500a400 (LWP 102226)] 0x0000000803ed64dc in ?? () from
/lib/libthr.so.3 [Switching to thread 2 (Thread 805007400 (LWP 101601))]
#0 0x00000008041f2afc in write () from /lib/libc.so.7
#0 0x00000008041f2afc in write () from /lib/libc.so.7 No symbol table info
available.
#1 0x0000000803ed0ce0 in write () from /lib/libthr.so.3 No symbol table info
available.
#2 0x0000000800864390 in ci_write (client=0x80bc4d000, msg=<optimized out>) at
clientbase.c:318
ap = {{gp_offset = 1648906823, fp_offset = 1499822669,
overflow_arg_area = 0x7665307538745955, reg_save_area = 0x6a6f5366550a0d50}}
cp = {{gp_offset = 4294942656, fp_offset = 32767, overflow_arg_area =
0x803ed57a4, reg_save_area = 0x7}}
t = <optimized out>
e = <optimized out>
n = 262143
left = 407634
s = 0x80af2a3c8
"iL5WIDKMjpjtT1iZSTGq7m6lhVOQx1uS0joFBOcj1FMmhE6SghWDkZAUD\r\nGP8A9VQ2BlLKb2/aGMo8hb94zHkHtk1N9nkmLyq+ZozgYflyOn1HvVJ6DLkMMaOoLbBnJIHr1oBN\r\ntc7cHY8gAYDO0Dv7VD1JYeSFLCR/MGc7j3yeKSa3Xy8gEHv701G47kEUM6zK4AP"...
buf = "717eCpXd2NNJn0jp8DW\r\nmmSCBti7cPuXOQeoFa"
__func__ = "ci_write"
#3 0x000000080086446c in ci_write_cb (client=0x80bc4d000) at clientbase.c:263
rest = <optimized out>
result = <optimized out>
#4 0x0000000800863095 in client_session_bailout (session=<optimized out>) at
clientsession.c:146
c = 0x805269c00
args = 0x0
from = 0x0
rcpt = 0x0
messagelst = 0x0
client = 0x80bc4d000
__func__ = "client_session_bailout"
#5 0x0000000000402285 in pop3_cb_time (arg=<optimized out>) at pop3.c:272
session = 0x805269c00
#6 0x0000000802961aee in event_base_loop () from
/usr/local/lib/event2/libevent-2.0.so.6
No symbol table info available.
#7 0x00000008008628a1 in server_run (conf=0x7fffffffa310) at server.c:842
i = 1
__func__ = "server_run"
#8 0x0000000800862d6d in server_mainloop (config=0x7fffffffa310,
service=0x403b3a "POP", servicename=<optimized out>) at server.c:976
__func__ = "server_mainloop"
#9 0x0000000000403a37 in main (argc=1, argv=<optimized out>) at pop3d.c:52
config = {no_daemonize = 0, log_verbose = 0, pidFile = 0x805017d80
"/var/run/dbmail/dbmail-pop3d.pid", timeout = 120, login_timeout = 30, iplist =
0x8050dd920, port = "110", '\000' <repeats 1020 times>, ssl_port = '\000'
<repeats 1023 times>, ipcount = 1, socketcount = 1, ssl_socketcount = 0,
listenSockets = 0x80500a800, ssl_listenSockets = 0x80500ac00,
service_before_smtp = 0, authlog = 0, ssl = 0, backlog = 512, resolveIP = 0,
evh = 0x0, service_name = "POP", '\000' <repeats 1020 times>, process_name =
"dbmail-pop3d", '\000' <repeats 1011 times>, serverUser = "nobody", '\000'
<repeats 1017 times>, serverGroup = "nogroup", '\000' <repeats 1016 times>,
socket = '\000' <repeats 1023 times>, log = "/var/log/dbmail.log", '\000'
<repeats 1004 times>, error_log = "/var/log/dbmail.err", '\000' <repeats 1004
times>, pid_dir = "/var/run/dbmail", '\000' <repeats 1008 times>, tls_cafile =
'\000' <repeats 1023 times>, tls_cert = '\000' <repeats 1023 times>, tls_key =
'\000' <rep
eats 1023 times>, tls_ciphers = '\000' <repeats 1023 times>, ClientHandler =
0x402060 <pop3_handle_connection>, cb = 0x0}
result = 0
__func__ = "main"
Best regards,
James Greig
-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of
Paul J Stevens
Sent: 17 March 2014 07:36
To: DBMail mailinglist
Subject: Re: [Dbmail] Iphone HTML emails
On 16-03-14 15:51, James Greig wrote:
> :D
>
> Getting somewhere now. I think basically attach gdb to the process and then
> run 'info threads' and switch to the second thread.
>
> :-
>
> Ok the below output was run whilst dbmail-pop3d was actually in an OK state
> but does the below actually look more like what is needed?
That's it exactly. bt full is probably overkill, but it doesn't hurt either...
--
________________________________________________________________
Paul J Stevens pjstevns @ gmail, twitter, github, linkedin
www.nfg.nl/[email protected]/+31.85.877.99.97
_______________________________________________
DBmail mailing list
[email protected]
http://mailman.fastxs.nl/cgi-bin/mailman/listinfo/dbmail
_______________________________________________
DBmail mailing list
[email protected]
http://mailman.fastxs.nl/cgi-bin/mailman/listinfo/dbmail
_______________________________________________
DBmail mailing list
[email protected]
http://mailman.fastxs.nl/cgi-bin/mailman/listinfo/dbmail
