I guess following is steps of the condition for the crash:
1/Current "numOpeners" of one well known stream = 2. This mean that just one
client open this stream. LOGD get close that stream request from client. The
closing stream is successful on active node. But checkpoiting to standby
have problem and cannot closing stream on standby.
2/Now the "numOpeners" on active node is (1), on standby node is (2). And
still have one client own that well known stream on standby although the
client already closed the stream on active node.
3/One other client open again that stream again. "numOpeners" on both active
node and standby is (2). Only one client own the stream, but 2 client own
that stream on standby.
4/Reboot active node and assume that 2 clients that is on active node, so 2
client will be downed. Standby node up to active will close that well known
stream 2 times with numOpeners = 2. Then numOpeners will be zero for the
well known stream. And the crash happens.
---
** [tickets:#3018] log: logd crash due to well known stream has numOpeners = 0**
**Status:** accepted
**Milestone:** 5.19.06
**Created:** Thu Mar 14, 2019 09:37 AM UTC by Canh Truong
**Last Updated:** Tue Mar 26, 2019 01:56 AM UTC
**Owner:** Canh Truong
osaflogd[14677]: src/log/logd/lgs_stream.cc:852: log_stream_close: Assertion
'stream->streamType >= STREAM_TYPE_APPLICATION_RT' failed.
osafamfnd[14733]: Rebooting OpenSAF NodeId = 131599 EE Name = , Reason:
Component faulted: recovery is node failfast, OwnNodeId = 131599,
SupervisionTime = 60
opensaf_reboot: Rebooting local node; timeout=60
opensaf_reboot: daemonize_job opensaf_kill returned=0
opensaf_reboot: opensaf_kill child process SUBPID=17690
(gdb) bt full
#0 0x00007f8938eeef67 in raise () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007f8938ef033a in abort () from /lib64/libc.so.6
No symbol table info available.
#2 0x00007f8939c6646e in __osafassert_fail (__file=<optimized out>,
__line=<optimized out>, __func=<optimized out>, __assertion=<optimized out>) at
src/base/sysf_def.c:296
No locals.
#3 0x0000558ff611c304 in log_stream_close (s=s@entry=0x7ffdf2741a78,
close_time_ptr=close_time_ptr@entry=0x7ffdf2741a48) at
src/log/logd/lgs_stream.cc:852
file_to_rename = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data
fields>},
members of std::basic_string<char, std::char_traits<char>,
std::allocator<char> >::_Alloc_hider:
_M_p = 0x7f8939a1dd58 <std::string::_Rep::_S_empty_rep_storage+24>
""
}
}
trace_num_openers = 0
closetime_tspec = {
tv_sec = 140728671148672,
tv_nsec = 140728671148560
}
root_path = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data
fields>},
members of std::basic_string<char, std::char_traits<char>,
std::allocator<char> >::_Alloc_hider:
_M_p = 0x558ff7867c28 "/storage/no-backup/coremw/var/log/saflog"
}
}
t_ = {
trace_leave_called = false,
file_ = 0x558ff61337ad "src/log/logd/lgs_stream.cc",
function_ = 0x558ff6134240 <log_stream_close(log_stream**,
long*)::__FUNCTION__> "log_stream_close"
}
errno_ret = 0
stream = 0x558ff78689d0
__FUNCTION__ = "log_stream_close"
rc = 0
timeString = 0x0
emptyStr = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data
fields>},
members of std::basic_string<char, std::char_traits<char>,
std::allocator<char> >::_Alloc_hider:
_M_p = 0x7f8939a1dd58 <std::string::_Rep::_S_empty_rep_storage+24>
""
}
}
#4 0x0000558ff60f2628 in lgs_client_delete (client_id=16,
closetime_ptr=closetime_ptr@entry=0x7ffdf2741b88) at src/log/logd/lgs_evt.cc:213
tmp_rec = <optimized out>
stream = 0x558ff78689d0
status = 0
cur_rec = 0x558ff786ab20
closetime = 1552070466
closetime_tspec = {
tv_sec = 2048,
tv_nsec = 140227354156352
}
client_rec = <optimized out>
__FUNCTION__ = "lgs_client_delete"
clientMap = 0x558ff7854490
client = 0x558ff786aa20
t_ = {
trace_leave_called = false,
file_ = 0x558ff612c49f "src/log/logd/lgs_evt.cc",
function_ = 0x558ff612d030 <lgs_client_delete(unsigned int,
long*)::__FUNCTION__> "lgs_client_delete"
}
#5 0x0000558ff60f2874 in lgs_client_delete_by_mds_dest
(mds_dest=586104137566712, closetime_ptr=closetime_ptr@entry=0x7ffdf2741b88) at
src/log/logd/lgs_evt.cc:343
it = {
_M_node = 0x558ff786aae0
}
rp = 0x558ff786aa20
__FUNCTION__ = "lgs_client_delete_by_mds_dest"
clientMap = 0x558ff7854490
rc = 0
t_ = {
trace_leave_called = false,
file_ = 0x558ff612c49f "src/log/logd/lgs_evt.cc",
function_ = 0x558ff612cfd0 <lgs_client_delete_by_mds_dest(unsigned
long, long*)::__FUNCTION__> "lgs_client_delete_by_mds_dest"
}
#6 0x0000558ff60f3020 in lgs_process_lga_down_list () at
src/log/logd/lgs_evt.cc:534
temp_lga_down_rec = 0x558ff786b0e0
closetime = 1552070466
lga_down_rec = 0x558ff786b0e0
closetime_tspec = {
tv_sec = 1552070466,
tv_nsec = 152364897
}
#7 proc_rda_cb_msg (evt=0x7f892c0008c0) at src/log/logd/lgs_evt.cc:577
count = <optimized out>
stream = <optimized out>
endloop = (unknown: 1552070466)
jstart = <optimized out>
rc = <optimized out>
t_ = {
trace_leave_called = false,
file_ = 0x558ff612c49f "src/log/logd/lgs_evt.cc",
function_ = 0x558ff612ce60
<proc_rda_cb_msg(lgsv_lgs_evt*)::__FUNCTION__> "proc_rda_cb_msg"
}
#8 lgs_process_mbx (mbx=<optimized out>) at src/log/logd/lgs_evt.cc:1568
msg = 0x7f892c0008c0
#9 0x0000558ff60e72ec in main (argc=<optimized out>, argv=<optimized out>) at
src/log/logd/lgs_main.cc:620
oi_handle = 0
oi_selection_object = <optimized out>
ret = <optimized out>
mbx_fd = <optimized out>
term_fd = 19
__FUNCTION__ = "main"
ais_rc = <optimized out>
rc = <optimized out>
cltimer_fd = -1
t_ = {
trace_leave_called = false,
file_ = 0x558ff6130a67 "src/log/logd/lgs_main.cc",
function_ = 0x558ff6130ebf <main::__FUNCTION__> "main"
}
---
Sent from sourceforge.net because [email protected] is
subscribed to https://sourceforge.net/p/opensaf/tickets/
To unsubscribe from further messages, a project admin can change settings at
https://sourceforge.net/p/opensaf/admin/tickets/options. Or, if this is a
mailing list, you can unsubscribe from the mailing list._______________________________________________
Opensaf-tickets mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/opensaf-tickets
