On 19/04/16 18:52, Willy Tarreau wrote:
On Tue, Apr 19, 2016 at 04:15:08PM +0200, Willy Tarreau wrote:
OK in fact it's different. Above we have a busy polling loop, which may
very be caused by the buffer space miscalculation bug and which results
in a process not completing its job until a timeout strikes. The link to
the other report shows a normal polling with blocked signals.
The processes that was created yesterday via soft reload, went 100% cpu
today.
haproxy 29388 5.0 0.0 58772 11700 ? Rs Apr17 156:44
/usr/sbin/haproxy -D -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid
-sf 1997
Section from strace output:
epoll_wait(0, {}, 200, 0) = 0
recvfrom(32,
"\366\334\247\270<\230\3028\v\334\236K\204^p\31\6\3T\230:\23s\257\337\316\242\302]\2\246\227"...,
15368, 0, NULL, NULL) = 15368
recvfrom(32,
"\366\334si\251\272Y\372\360'/\363\212\246\262w\307[\251\375\314\236whe\302\337\257\25NQ\370"...,
1024, 0, NULL, NULL) = 1024
sendto(18,
"\366\334\247\270<\230\3028\v\334\236K\204^p\31\6\3T\230:\23s\257\337\316\242\302]\2\246\227"...,
16392, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_MORE, NULL, 0) = 8016
sendto(18,
"\355\265\207\360\357\3046k\364\320\330\30d\247\354\273BE\201\337\4\265#\357Z\231\231\337\365*\242\345"...,
8376, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_MORE, NULL, 0) = -1 EAGAIN (Resource
temporarily unavailable)
epoll_ctl(0, EPOLL_CTL_MOD, 18, {EPOLLIN|EPOLLOUT|EPOLLRDHUP, {u32=18,
u64=18}}) = 0
epoll_wait(0, {}, 200, 0) = 0
recvfrom(32,
"@OR\224\335\233\263\347U\245X\376)\240\342\334\242\31\321\322\354\222\276\233\247\316-\263\370)\252U"...,
8016, 0, NULL, NULL) = 8016
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {{EPOLLOUT, {u32=53, u64=53}}}, 200, 0) = 1
sendto(53,
"\274'[\24\n\264*b\306\253YA\313A\36\202a\177\317\370K:\302\230\315.\315\215\f&\351\27"...,
14032, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_MORE, NULL, 0) = 14032
sendto(53,
"\234CS\236wYsf\267\24\276v\325\302\267+a\303\336\250\211x\236\33\23MR_\324\214A\264"...,
2360, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_MORE, NULL, 0) = 2360
recvfrom(55,
"\231\16\35\337\20\203V\344\360\202n\307\2120\213\r\353\312\334\357\205\366=\\\373|\210\4-\354\32\360"...,
15368, 0, NULL, NULL) = 15368
recvfrom(55,
"i\244\305N\242I\177n'4g\211\256%\26X\34il\3374\34HN\22\365\357\211Y\354\306K"...,
1024, 0, NULL, NULL) = 1024
sendto(53,
"\231\16\35\337\20\203V\344\360\202n\307\2120\213\r\353\312\334\357\205\366=\\\373|\210\4-\354\32\360"...,
16392, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_MORE, NULL, 0) = 16392
epoll_ctl(0, EPOLL_CTL_MOD, 53, {EPOLLIN|EPOLLRDHUP, {u32=53, u64=53}}) = 0
epoll_wait(0, {}, 200, 0) = 0
recvfrom(55,
"\365f\303r(\1\365S\276\246c\334\216\346\226\10<}\340\227h\374\370\360\276sSs\346\351\337\370"...,
15368, 0, NULL, NULL) = 15368
recvfrom(55,
"-\r\21\326\326\0\0>\346-?\375\325J\346N\336\353Jz\376\303\373?\226y}\317\257\371\304t"...,
1024, 0, NULL, NULL) = 1024
sendto(53,
"\365f\303r(\1\365S\276\246c\334\216\346\226\10<}\340\227h\374\370\360\276sSs\346\351\337\370"...,
16392, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_MORE, NULL, 0) = 16392
epoll_wait(0, {}, 200, 0) = 0
recvfrom(55,
"\251\3\0200\317\217ab\223\f\306\322/}J\231\4\3b\311h\220sq\220[\225\21\372\264Dv"...,
15368, 0, NULL, NULL) = 15368
recvfrom(55,
"\233.\20B\337\343\274\311\212\211\241\244\5\257\221w1{\253Kjh\23?w\357\365\377\335\261\3\215"...,
1024, 0, NULL, NULL) = 1024
sendto(53,
"\251\3\0200\317\217ab\223\f\306\322/}J\231\4\3b\311h\220sq\220[\225\21\372\264Dv"...,
16392, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_MORE, NULL, 0) = 16392
epoll_wait(0, {}, 200, 0) = 0
recvfrom(55,
",T\27\22\300\31\231t\207%j-\263}\344\25#\333\235\214*M\227\26\0215*_\312/@\351"...,
15368, 0, NULL, NULL) = 15368
recvfrom(55,
"\225\256\37Qib\371\377\220l\342\20\2742\271\3360U\224\0375?ju\10\207\235J\267\35\340\367"...,
1024, 0, NULL, NULL) = 1024
sendto(53,
",T\27\22\300\31\231t\207%j-\263}\344\25#\333\235\214*M\227\26\0215*_\312/@\351"...,
16392, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_MORE, NULL, 0) = 13312
sendto(53,
"\372\265\334\263\232\2016l2\216\372\261B\26\243\252\204\220\353\f\367\215\331\232\203hI,\260\37\207\357"...,
3080, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_MORE, NULL, 0) = -1 EAGAIN (Resource
temporarily unavailable)
epoll_ctl(0, EPOLL_CTL_MOD, 53, {EPOLLIN|EPOLLOUT|EPOLLRDHUP, {u32=53,
u64=53}}) = 0
epoll_wait(0, {}, 200, 0) = 0
recvfrom(55,
"k\33\342U\260:Z\350\3725>\211R@\20\347\326\363\203\36?\226\304\241\367\263B\242\230\6^\221"...,
13312, 0, NULL, NULL) = 13312
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {{EPOLLIN, {u32=1, u64=1}}}, 200, 0) = 1
recvfrom(1, "148112 syslog 341 <174>2016-04-1"..., 15368, 0, NULL, NULL)
= 360
write(2,
"\27\3\3\1\200\f\364\2630\5\334\372(\223e8\347\34\302(\277\216=\264\256N\365\1Y\204%\335"...,
389) = 389
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
epoll_wait(0, {}, 200, 0) = 0
Veiko
