Hi, Daniel!
Are you doing any "expensive" processing on reply messages? Or in
failure route? Any DB or REST queries?
It looks like the process is waiting for a lock in the transaction
module, which usually means that the transaction is stuck doing
something else.
Best regards,
Răzvan Crainea
OpenSIPS Solutions
www.opensips-solutions.com
On 02/21/2017 05:43 PM, Daniel Zanutti wrote:
Hi
We are having a weird issue on a high load server, could you guys
please help me find out a solution?
These opensips threads freeze for a few seconds at 100% CPU and drop
some packets. So some new calls fails to be processed:
Inline image 2
Running on a Debian 8.6 x64 - 12 cores/48GB Ram server:
# opensips -V
version: opensips 1.11.6-notls (x86_64/linux)
flags: STATS: On, USE_IPV6, USE_TCP, DISABLE_NAGLE, USE_MCAST,
SHM_MEM, SHM_MMAP, PKG_MALLOC, F_MALLOC, FAST_LOCK-ADAPTIVE_WAIT
ADAPTIVE_WAIT_LOOPS=1024, MAX_RECV_BUFFER_SIZE 262144, MAX_LISTEN 16,
MAX_URI_SIZE 1024, BUF_SIZE 65535
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
main.c compiled on 10:46:04 Oct 20 2016 with gcc 4.9.2
I got the gdb of one of one frozen thread, it's seems to be something
related to fastlock:
(gdb) bt full
#0 0x00007ff4cc9e4bb7 in sched_yield () at
../sysdeps/unix/syscall-template.S:81
No locals.
#1 0x00007ff4cacd9d5d in get_lock (lock=0x7ff3cb6e2678) at
../../evi/../fastlock.h:182
i = 0
#2 _lock (s=0x7ff3cb6e2678) at lock.h:102
No locals.
#3 reply_received (p_msg=0x7ff4cc809c38) at t_reply.c:1556
msg_status = 302
last_uac_status = 503
branch = 1
reply_status = <optimized out>
timer = 140689351533517
cancel_bitmap = 0
uac = <optimized out>
t = 0x7ff3cb6e0e20
backup_list = <optimized out>
has_reply_route = 1
__FUNCTION__ = "reply_received"
#4 0x000000000044d613 in forward_reply (msg=msg@entry=0x7ff4cc809c38)
at forward.c:575
new_buf = 0x0
to = 0x0
new_len = 32755
mod = 0x7ff4cc71b2b0
proto = <optimized out>
id = 0
send_sock = <optimized out>
s = <optimized out>
len = <optimized out>
__FUNCTION__ = "forward_reply"
#5 0x00000000004340ff in receive_msg (buf=<optimized out>,
len=<optimized out>, rcv_info=rcv_info@entry=0x7fff9f1182f0) at
receive.c:214
msg = 0x7ff4cc809c38
start = {tv_sec = 140735862113030, tv_usec = 140689389048549}
rc = 3
__FUNCTION__ = "receive_msg"
#6 0x0000000000462ef4 in udp_rcv_loop () at udp_server.c:422
len = <optimized out>
buf = "SIP/2.0 302 Moved Temporarily\r\nVia: SIP/2.0/UDP
2.2.2.2:5060;rport=5060;received=2.2.2.2;branch=z9hG4bK9ef.5fb57785.1\r\nVia:
SIP/2.0/UDP 1.1.1.1:5060;rport=5060;received=1.1.1"...
from = 0x7ff4cc809c00
fromlen = 16
ri = {src_ip = {af = 2, len = 4, u = {addrl = {41876154, 1},
addr32 = {41876154, 0, 1, 0}, addr16 = {64186, 638, 0, 0, 1, 0, 0, 0},
addr = "\272\372~\002\000\000\000\000\001\000\000\000\000\000\000"}},
dst_ip = {af = 2,
len = 4, u = {addrl = {769317832, 0}, addr32 = {769317832,
0, 0, 0}, addr16 = {56264, 11738, 0, 0, 0, 0, 0, 0}, addr =
"\310\333\332-", '\000' <repeats 11 times>}}, src_port = 5060,
dst_port = 5060, proto = 1,
proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s =
{sa_family = 2, sa_data =
"\023ĺ\372~\002\000\000\000\000\000\000\000"}, sin = {sin_family = 2,
sin_port = 50195, sin_addr = {s_addr = 41876154},
sin_zero = "\000\000\000\000\000\000\000"}, sin6 =
{sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 41876154,
sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>,
__u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0},
__u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}},
bind_address = 0x7ff4cc71ade0}
p = <optimized out>
msg = {
s = 0x81e780 <buf> "SIP/2.0 302 Moved Temporarily\r\nVia:
SIP/2.0/UDP
2.2.2.2:5060;rport=5060;received=2.2.2.2;branch=z9hG4bK9ef.5fb57785.1\r\nVia:
SIP/2.0/UDP 1.1.1.1:5060;rport=5060;received=1.1.1"..., len = 537}
__FUNCTION__ = "udp_rcv_loop"
#7 0x00000000004161e0 in main_loop () at main.c:929
i = 3
pid = <optimized out>
si = 0x7ff4cc71ade0
startup_done = 0x0
chd_rank = 4
rc = <optimized out>
load_p = 0x7ff3c61ac430
#8 main (argc=<optimized out>, argv=<optimized out>) at main.c:1613
cfg_log_stderr = <optimized out>
cfg_stream = <optimized out>
c = <optimized out>
r = <optimized out>
---Type <return> to continue, or q <return> to quit---
tmp = 0x7fff9f118efe ""
tmp_len = <optimized out>
port = <optimized out>
proto = <optimized out>
options = 0x5c0aa8 "f:cCm:M:b:l:n:N:rRvdDFETSVhw:t:u:g:P:G:W:o:"
ret = -1
seed = 3165191676
rfd = <optimized out>
__FUNCTION__ = "main"
Any idea?
Thanks!
_______________________________________________
Users mailing list
Users@lists.opensips.org
http://lists.opensips.org/cgi-bin/mailman/listinfo/users
_______________________________________________
Users mailing list
Users@lists.opensips.org
http://lists.opensips.org/cgi-bin/mailman/listinfo/users