Hi all I have a setup of several proxies behind a load balancer and all of them have several coredumps every day. I've tried versions 5.3, 5.5 and 5.8 and all of them crash
I didn't have dbgsym packages installed but now I've built a 5.8.4 version in one of them and can gdb the coredumps. Can't see anything particular and I'm wondering if I'm doing anything wrong. I'm descompressing the coredump with lz4cat. Sometimes I see that the system generates 3 coredumps at the same time. I've tried running gdb in one of them and exec "bt full" I usually see "can't access memory" in some of them but inb others I don't see anything relevant. I'm attaching one coredump. I don't even know if I'm doing it properly. Could you please guide me to how to debug what's going on? thanks
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/sbin/kamailio -P /run/kamailio/kamailio.proxy.pid
-f /etc/kamailio/proxy/k'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 search_next_avp (s=s@entry=0x7f69566d1d60 <st>,
val=val@entry=0x7fff9b242d30) at core/usr_avp.c:515
515 core/usr_avp.c: No such file or directory.
[Current thread is 1 (Thread 0x7f6965450100 (LWP 19950))]
(gdb) bt full
#0 search_next_avp (s=s@entry=0x7f69566d1d60 <st>,
val=val@entry=0x7fff9b242d30) at core/usr_avp.c:515
matched = <optimized out>
avp = <optimized out>
list = <optimized out>
__func__ = "search_next_avp"
#1 0x00007f69566a4265 in legs2strar (legs=0x7f695b3ca200,
rq=rq@entry=0x7f676470bc90, val_arr=<optimized out>,
int_arr=0x7f695b3c6de4, type_arr=0x7f695b3c6f9d "\002\002\001",
start=start@entry=0) at acc_extra.c:363
avp = {0x7f67565bd430, 0x7f675ec27600, 0x7f675ec27550, 0x0 <repeats 13
times>}
st = {{flags = 272, id = 0, name = {n = 901, s = {s = 0x385 <error:
Cannot access memory at address 0x385>, len = 0},
re = 0x385}, avp = 0x6638662d65313866}, {flags = 272, id = 0,
name = {n = 902, s = {
s = 0x386 <error: Cannot access memory at address 0x386>, len =
0}, re = 0x386}, avp = 0x7f675ec27550}, {
flags = 273, id = 120, name = {n = 140090447689591, s = {s =
0x7f69595ef777 "acc_branch_id)", len = 13},
re = 0x7f69595ef777}, avp = 0x7f675ec274a0}, {flags = 0, id = 0,
name = {n = 0, s = {s = 0x0, len = 0}, re = 0x0},
avp = 0x0} <repeats 13 times>}
name_type = 0
name = {n = 140090439056104, s = {s = 0x7f6958db3ae8 "entry", len = 5},
re = 0x7f6958db3ae8}
value = {n = 140090479241840, s = {s = 0x7f695b406a70 "0", len = 1}, re
= 0x7f695b406a70}
n = 0
found = <optimized out>
r = 0
#2 0x00007f695667ff07 in acc_db_request (rq=rq@entry=0x7f676470bc90) at
acc.c:526
m = 13
n = <optimized out>
i = <optimized out>
o = 5
t = {tm_sec = 1, tm_min = 0, tm_hour = -1880927744, tm_mday =
-1876427672, tm_mon = -1692126896, tm_year = 32767,
tm_wday = 1685111456, tm_yday = 32615, tm_isdst = 1685115376,
tm_gmtoff = 140082043473448, tm_zone = 0x0}
dtime = <optimized out>
__func__ = "acc_db_request"
#3 0x00007f695668f3d9 in acc_onreply (code=<optimized out>, reply=<optimized
out>, req=<optimized out>, t=<optimized out>)
at acc_logic.c:635
br = <optimized out>
cmsg = 0x7f676470bc90
cmsg_len = 8680
preq = 0x7f676470bc90
--Type <RET> for more, q to quit, c to continue without paging--c
mend = 0x7f676470de78
new_uri_bk = {s = 0x7f676470c470
"sip:[email protected]:5060;transport=udp", len = 49}
hdr = <optimized out>
mstart = 0x7f676470bc90
new_uri_bk = <optimized out>
br = <optimized out>
hdr = <optimized out>
cmsg = <optimized out>
cmsg_len = <optimized out>
preq = <optimized out>
mstart = <optimized out>
mend = <optimized out>
__func__ = "acc_onreply"
__llevel = <optimized out>
__kld = <optimized out>
__llevel = <optimized out>
__kld = <optimized out>
__llevel = <optimized out>
__kld = <optimized out>
#4 tmcb_func (t=<optimized out>, type=<optimized out>, ps=<optimized out>) at
acc_logic.c:725
__func__ = "tmcb_func"
#5 0x00007f6958c8f6fb in run_trans_callbacks_internal (cb_lst=<optimized out>,
type=type@entry=512, trans=0x7f6761012a90, params=params@entry=0x7fff9b243150)
at t_hooks.c:236
cbp = 0x7f675ec279a0
backup_xd = {uri_avps_from = 0x555d20a72020 <def_list>, uri_avps_to =
0x555d20a72028 <def_list+8>, user_avps_from = 0x555d20a72030 <def_list+16>,
user_avps_to = 0x555d20a72038 <def_list+24>, domain_avps_from = 0x555d20a72040
<def_list+32>, domain_avps_to = 0x555d20a72048 <def_list+40>, xavps_list =
0x555d20a715d8 <_xavp_list_head>, xavus_list = 0x555d20a715d0
<_xavu_list_head>, xavis_list = 0x555d20a715c8 <_xavi_list_head>}
__func__ = "run_trans_callbacks_internal"
#6 0x00007f6958c8fa52 in run_trans_callbacks_with_buf (type=type@entry=512,
rbuf=rbuf@entry=0x7f6761012b90, req=<optimized out>,
repl=repl@entry=0x7f695b404a60, flags=flags@entry=0) at t_hooks.c:272
params = {req = 0x7f67560da8e0, rpl = 0x7f695b404a60, param =
0x7f675ec279b0, code = 487, flags = 0, branch = 0, t_rbuf = 0x7f6761012b90, dst
= 0x7f6761012be0, send_buf = {s = 0x7f675671c760
"<sip:ping@localhost>;tag=6268cbfa57ab9925939f7db74a7e8768-ab55be17", len =
852}}
trans = <optimized out>
#7 0x00007f6958c55850 in relay_reply (t=t@entry=0x7f6761012a90,
p_msg=p_msg@entry=0x7f695b404a60, branch=<optimized out>,
msg_status=msg_status@entry=487, cancel_data=cancel_data@entry=0x7fff9b243470,
do_put_on_wait=do_put_on_wait@entry=1) at t_reply.c:2158
relay = 0
save_clone = 0
buf = 0x7f695b404130 "SIP/2.0 487 Request terminated\r\nRecord-Route:
<sip:192.168.254.103:5062;lr=on;ftag=S7DSyH201U3Nc;did=2ca.8061;ice_caller=strip;ice_callee=strip;aset=50>\r\nRecord-Route:
<sip:192.168.254.202;r2=on;lr=on"...
res_len = 852
relayed_code = 487
relayed_msg = 0x7f695b404a60
reply_bak = <optimized out>
bm = {to_tag_val = {s = 0x1e7 <error: Cannot access memory at address
0x1e7>, len = 543834009}}
totag_retr = <optimized out>
reply_status = <optimized out>
uas_rb = 0x7f6761012b90
to_tag = <optimized out>
reason = {s = 0x7f676470b7e0 "\020\231pdg\177", len = 83}
onsend_params = {req = 0x7f695b404a60, rpl = 0x2, param =
0x7f676470b820, code = 1627467104, flags = 32615, branch = 0, t_rbuf = 0x1e7,
dst = 0x9027fc688fe35200, send_buf = {s = 0x7f695b3b1c00 "\257\004", len =
1440774528}}
ip = {af = 1440505856, len = 32615, u = {addrl = {140082043467808,
140090438110517}, addr32 = {1685108768, 32615, 1489816885, 32617}, addr16 =
{47136, 25712, 32615, 0, 52533, 22732, 32617, 0}, addr = "
\270pdg\177\000\000\065\315\314Xi\177\000"}}
__func__ = "relay_reply"
#8 0x00007f6958c571b3 in reply_received (p_msg=0x7f695b404a60) at
t_reply.c:2719
msg_status = 487
last_uac_status = 180
ack = <optimized out>
ack_len = 399
branch = 0
reply_status = <optimized out>
onreply_route = <optimized out>
cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s =
0x0, len = 0}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 0}}}}
uac = 0x7f6761012d60
t = 0x7f6761012a90
lack_dst = {send_sock = 0x555d2095b0b0, to = {s = {sa_family = 62342,
sa_data = "AVi\177\000\000`5$\233\377\177\000"}, sin = {sin_family = 62342,
sin_port = 22081, sin_addr = {s_addr = 32617}, sin_zero =
"`5$\233\377\177\000"}, sin6 = {sin6_family = 62342, sin6_port = 22081,
sin6_flowinfo = 32617, sin6_addr = {__in6_u = {__u6_addr8 =
"`5$\233\377\177\000\000z\316\063\233\377\177\000", __u6_addr16 = {13664,
39716, 32767, 0, 52858, 39731, 32767, 0}, __u6_addr32 = {2602841440, 32767,
2603863674, 32767}}}, sin6_scope_id = 26502688}, sas = {ss_family = 62342,
__ss_padding =
"AVi\177\000\000`5$\233\377\177\000\000z\316\063\233\377\177\000\000
f\224\001\000\000\000\000`\023;[\001\000\000\000`\023;[i\177\000\000\004\240\214
]U\000\000\340\071$\233\377\177\000\000\062\r\217
]U\000\000\001\000\000\000\000\000\000\000\066Mi
]U\000\000\340\071$\233\377\177\000\000`\023;[i\177\000\000`J@[i\177\000\000\004\240\214
]U\000", __ss_align = 93858466406404}}, id = 546269776, send_flags = {f =
21853, blst_imask = 0}, proto = 0 '\000', proto_pad0 = 0 '\000', proto_pad1 = 0}
backup_user_from = <optimized out>
backup_user_to = <optimized out>
backup_domain_from = <optimized out>
backup_domain_to = <optimized out>
backup_uri_from = <optimized out>
backup_uri_to = <optimized out>
backup_xavps = <optimized out>
backup_xavus = <optimized out>
backup_xavis = <optimized out>
replies_locked = 1
branch_ret = <optimized out>
prev_branch = <optimized out>
failover_continue = <optimized out>
blst_503_timeout = <optimized out>
hf = <optimized out>
onsend_params = {req = 0x555d208f0d32, rpl = 0x7f695641f386
<pkg_proc_update_stats+70>, param = 0xc000000, code = 1447162758, flags =
32617, branch = 0, t_rbuf = 0xc000000, dst = 0xa035ec0, send_buf = {s =
0x1946de0 <error: Cannot access memory at address 0x1946de0>, len = 33333568}}
ctx = {rec_lev = 0, run_flags = 1, last_retcode = 1, jmp_env =
{{__jmpbuf = {140735796229552, 8124283468966394013, 140090466511088,
140090479233632, 140081985826144, 487, -8124363379020007267,
-2736781096255489891}, __mask_was_saved = 0, __saved_mask = {__val =
{10387548591574176256, 140090473604080, 93858467701384, 6, 93858467701449,
140090473604144, 73, 93858467701377, 93858465510692, 4, 0, 1738041542, 818674,
1738041542, 818713, 201326592}}}}}
bctx = <optimized out>
keng = <optimized out>
ret = <optimized out>
evname = {s = 0x7f6958ccd9c4 "on_sl_reply", len = 11}
__func__ = "reply_received"
#9 0x0000555d2062aa6c in do_forward_reply (msg=msg@entry=0x7f695b404a60,
mode=mode@entry=0) at core/forward.c:786
new_buf = 0x0
dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000'
<repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr =
0}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 0,
sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = '\000'
<repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0,
0, 0, 0}}}, sin6_scope_id = 0}, sas = {ss_family = 0, __ss_padding = '\000'
<repeats 117 times>, __ss_align = 0}}, id = 0, send_flags = {f = 0, blst_imask
= 0}, proto = 0 '\000', proto_pad0 = 0 '\000', proto_pad1 = 0}
new_len = 21853
r = 2
ip = {af = 1530940000, len = 32617, u = {addrl = {93858467701363,
93858467701449}, addr32 = {547381875, 21853, 547381961, 21853}, addr16 =
{25203, 8352, 21853, 0, 25289, 8352, 21853, 0}, addr = "sb\240
]U\000\000\311b\240 ]U\000"}}
s = <optimized out>
len = <optimized out>
__func__ = "do_forward_reply"
#10 0x0000555d2062d0a7 in forward_reply (msg=msg@entry=0x7f695b404a60) at
core/forward.c:883
No locals.
#11 0x0000555d206ba04c in receive_msg (buf=buf@entry=0x555d20a05f40 <buf>
"SIP/2.0 487 Request terminated\r\nRecord-Route:
<sip:192.168.254.103:5062;lr=on;ftag=S7DSyH201U3Nc;did=2ca.8061;ice_caller=strip;ice_callee=strip;aset=50>\r\nRecord-Route:
<sip:192.168.254.202;r2=on;lr=on"..., len=<optimized out>,
rcv_info=rcv_info@entry=0x7fff9b243c00) at core/receive.c:616
msg = 0x7f695b404a60
ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env =
{{__jmpbuf = {140735796230624, 8124283468966394013, 140090478891872,
140090479233632, 0, 0, -8124363379198265187, -2736781096255489891},
__mask_was_saved = 0, __saved_mask = {__val = {0, 93858466451073,
10322380694930716829, 1, 0, 21, 140081796078969, 28, 140081796046382,
18446744073709551536, 93858481594896, 93858481594896, 93858481594896, 0,
140090649225366, 140081805102760}}}}}
bctx = <optimized out>
ret = <optimized out>
tvb = {tv_sec = 0, tv_usec = 0}
tve = {tv_sec = 0, tv_usec = 0}
diff = 0
inb = {s = 0x555d20a05f40 <buf> "SIP/2.0 487 Request
terminated\r\nRecord-Route:
<sip:192.168.254.103:5062;lr=on;ftag=S7DSyH201U3Nc;did=2ca.8061;ice_caller=strip;ice_callee=strip;aset=50>\r\nRecord-Route:
<sip:192.168.254.202;r2=on;lr=on"..., len = 905}
netinfo = {data = {s = 0x0, len = 0}, bufsize = 0, rcv = 0x0, dst = 0x0}
keng = <optimized out>
evp = {data = 0x7fff9b243920, obuf = {s = 0x0, len = 0}, rcv =
0x7fff9b243c00, dst = 0x0, req = 0x0, rpl = 0x0, rplcode = 0, mode = 0}
cidlockidx = <optimized out>
cidlockset = <optimized out>
errsipmsg = 0
exectime = 0
__func__ = "receive_msg"
#12 0x0000555d2058bff2 in udp_rcv_loop () at core/udp_server.c:760
len = 905
buf = "SIP/2.0 487 Request terminated\r\nRecord-Route:
<sip:192.168.254.103:5062;lr=on;ftag=S7DSyH201U3Nc;did=2ca.8061;ice_caller=strip;ice_callee=strip;aset=50>\r\nRecord-Route:
<sip:192.168.254.202;r2=on;lr=on"...
tmp = <optimized out>
fromaddr = <optimized out>
fromaddrlen = 16
rcvi = {src_ip = {af = 2, len = 4, u = {addrl = {1744742592, 0}, addr32
= {1744742592, 0, 0, 0}, addr16 = {43200, 26622, 0, 0, 0, 0, 0, 0}, addr =
"\300\250\376g", '\000' <repeats 11 times>}}, dst_ip = {af = 2, len = 4, u =
{addrl = {1744742592, 0}, addr32 = {1744742592, 0, 0, 0}, addr16 = {43200,
26622, 0, 0, 0, 0, 0, 0}, addr = "\300\250\376g", '\000' <repeats 11 times>}},
src_port = 5080, dst_port = 5062, proto_reserved1 = 0, proto_reserved2 = 0,
src_su = {s = {sa_family = 2, sa_data =
"\023\330\300\250\376g\000\000\000\000\000\000\000"}, sin = {sin_family = 2,
sin_port = 55315, sin_addr = {s_addr = 1744742592}, sin_zero =
"\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 55315,
sin6_flowinfo = 1744742592, sin6_addr = {__in6_u = {__u6_addr8 = '\000'
<repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0,
0, 0, 0}}}, sin6_scope_id = 0}, sas = {ss_family = 2, __ss_padding =
"\023\330\300\250\376g", '\000' <repeats 111 times>, __ss_align = 0}},
bind_address = 0x7f69595bf2c0, rflags = (unknown: 0), proto = 1 '\001',
proto_pad0 = 0 '\000', proto_pad1 = 0}
evp = {data = 0x0, obuf = {s = 0x0, len = 0}, rcv = 0x0, dst = 0x0, req
= 0x0, rpl = 0x0, rplcode = 0, mode = 0}
printbuf =
"\250\216;Vg\177\000\000\n\000\000\000\000\000\000\000\025\000\000\000\000\000\000\000x8c
]U\000\000\004\000\000\000\000\000\000\000\004>$\233\377\177\000\000\n\000\000\000\000\000\000\000\322Æ
]U\000\000\000\000\000\000\000\000\000\000\060\236\256
]U\000\000^D\324\004\362o\247\351\346\235\370\371\356j\254\241u\031}\026k\326c!\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\020\236\256
]U\000\000Ð
\235<\000\000\000\000\070>$\233\377\177\000\000\"\225\005\325\000\000\000\000\000R\343\217h\374'\220\"\225\005\325\000\000\000\000\211\000\000\000\000\000\000\000\275\247\216
]U\000\000\n\000\000\000\000\000\000\000\250\216;Vg\177\000\000\n\000\000\000\000\000\000\000"...
i = <optimized out>
j = <optimized out>
l = <optimized out>
__func__ = "udp_rcv_loop"
__llevel = <optimized out>
__kld = <optimized out>
#13 0x0000555d20548a70 in main_loop () at main.c:1815
i = <optimized out>
pid = <optimized out>
si = <optimized out>
si_desc = "udp receiver child=9
sock=192.168.254.103:5062\000\000\001\000\000\000\000\000\000\000\001\205c
]U\000\000H \247
]U\000\000\000R\343\217h\374'\220\001\000\000\000\000\000\000\000\000R\343\217h\374'\220\340\062\240
]U\000\000\000R\343\217h\374'\220\b|\256 ]U\000\000\b\000\000\000\000\000\000"
nrprocs = <optimized out>
woneinit = 1
__func__ = "main_loop"
#14 0x0000555d2053edc2 in main (argc=<optimized out>, argv=<optimized out>) at
main.c:3256
cfg_stream = <optimized out>
c = <optimized out>
r = <optimized out>
tmp = 0x7fff9b245e9e ""
tmp_len = 0
port = 5060
proto = 0
aproto = 0
ahost = 0x0
aport = 0
options = 0x555d208ccb40
":f:cm:M:dVIhEeb:B:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 4208105427
rfd = <optimized out>
debug_save = <optimized out>
debug_flag = <optimized out>
dont_fork_cnt = <optimized out>
n_lst = <optimized out>
p = <optimized out>
st = {st_dev = 21, st_ino = 26688, st_nlink = 2, st_mode = 16872,
st_uid = 111, st_gid = 118, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize =
4096, st_blocks = 0, st_atim = {tv_sec = 1732637887, tv_nsec = 632097068},
st_mtim = {tv_sec = 1737990795, tv_nsec = 514354175}, st_ctim = {tv_sec =
1737990841, tv_nsec = 326361374}, __glibc_reserved = {0, 0, 0}}
l1 = <optimized out>
tbuf =
"\000\000\000\000\000\000\000\000\b\000\000\000\000\000\000\000\060Çei\177\000\000\000\000\000\000\000\000\000\000\240\301\063\233\377\177\000\000\004\313~ei\177\000\000\b\000\000\000\000\000\000\000\230\302\063\233\377\177\000\000\060Çei\177\000\000(C$\233\377\177\000\000$C$\233\377\177",
'\000' <repeats 18 times>,
"\230\302\063\233\377\177\000\000\250\301\063\233\377\177\000\000'\016]ei\177\000\000&\260be\000\000\000\000\220D$\233\377\177\000\000\300\212\225\001\000\000\000\000\360C$\233\377\177\000\000\340C$\233\377\177\000\000(C$\233\377\177\000\000XÇei\177\000\000\001",
'\000' <repeats 31 times>...
option_index = 0
long_options = {{name = 0x555d208cb25a "help", has_arg = 0, flag = 0x0,
val = 104}, {name = 0x555d209134ad "version", has_arg = 0, flag = 0x0, val =
118}, {name = 0x555d208d8fae "alias", has_arg = 1, flag = 0x0, val = 1024},
{name = 0x555d208cb25f "subst", has_arg = 1, flag = 0x0, val = 1025}, {name =
0x555d208cb265 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name =
0x555d208cb26e "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name =
0x555d208cb278 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name =
0x555d208cb282 "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name =
0x555d208cb28d "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name =
0x555d208cb296 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name =
0x555d209135ca "debug", has_arg = 1, flag = 0x0, val = 1032}, {name =
0x555d208cb2a1 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name =
0x555d208cb2ab "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name =
0x555d208cb2b2 "all-errors", has_arg = 0, flag = 0x0, val = 1035}, {name = 0x0,
has_arg = 0, flag = 0x0, val = 0}}
__func__ = "main"
(gdb)
pgpNW4hJ027GA.pgp
Description: Firma digital OpenPGP
__________________________________________________________ Kamailio - Users Mailing List - Non Commercial Discussions -- [email protected] To unsubscribe send an email to [email protected] Important: keep the mailing list in the recipients, do not reply only to the sender!
