Hi All,
We are facing a random crash during the scale of MPLS tunnels(8000 mpls
tunnels). The crash has been observed multiple times and the call stack is
the same.
During the worker thread crash the main thread has executed the following
lines of code(between the barrier sync and release), please provide us
inputs to debug this issue.
vlib_worker_thread_barrier_sync (vm);
ret_val = rtb_vpp_route_rpath_create(route_mapping, &rpath,
RTB_VPP_ADD);
if (RTB_VPP_SUCCESS != ret_val) {
goto Exit;
}
vec_add1(rpaths, rpath);
tunnel_sw_if_index = vnet_mpls_tunnel_create(1, 0, NULL);
vnet_mpls_tunnel_path_add(tunnel_sw_if_index, rpaths);
vlib_worker_thread_barrier_release (vm);
VPP Version : 20.09
BT:
Thread 3 (Thread 0x7fcf87fff700 (LWP 22778)):
#0 0x00007fd04613a492 in __GI___waitpid (pid=25786,
stat_loc=stat_loc@entry=0x7fd02514b9a8,
options=options@entry=0) at ../sysdeps/unix/sysv/linux/waitpid.c:30
#1 0x00007fd0460a5177 in do_system (line=line@entry=0x7fd02514bb30
"/usr/local/bin/rtb-dump-core.sh '22729' 'fibd'") at
../sysdeps/posix/system.c:149
#2
* 0x00007fd0460a555a in __libc_system (line=line@entry=0x7fd02514bb30
"/usr/local/bin/rtb-dump-core.sh '22729' 'fibd'") at
../sysdeps/posix/system.c:185#3 0x00007fd04667105d in bd_signal_handler_cb
(signo=11) at
/development/rtbrick-infrastructure/code/bd/src/bdinfra/bd.c:753#4
0x00007fd037b4b73f in rtb_bd_signal_handler (signo=11) at
/development/libvpp/src/vlib/unix/main.c:80#5 0x00007fd037b4bba2 in
unix_signal_handler (signum=11, si=0x7fd02514bfb0, uc=0x7fd02514be80) at
/development/libvpp/src/vlib/unix/main.c:180#6 <signal handler called>#7
0x00007fd0372f1389 in mspace_is_heap_object (msp=0x7fd008ec3010,
p=0x7fd0264c66c8) at /development/libvpp/src/vppinfra/dlmalloc.c:4134#8
0x00007fd037b1b482 in clib_mem_is_heap_object (p=0x7fd0264c66c8) at
/development/libvpp/src/vppinfra/mem.h:211#9 0x00007fd037b0aaa6 in
_vec_resize_inline (v=0x7fd0264c66d0, length_increment=1, data_bytes=59368,
header_bytes=0, data_align=8, numa_id=255) at
/development/libvpp/src/vppinfra/vec.h:154#10 0x00007fd037b15fbe in
vlib_worker_thread_node_refork () at
/development/libvpp/src/vlib/threads.c:1133#11 0x00007fd037ac8196 in
vlib_worker_thread_barrier_check () at
/development/libvpp/src/vlib/threads.h:482#12 0x00007fd037ac259e in
vlib_main_or_worker_loop (vm=0x7fd00c58ac40, is_main=0) at
/development/libvpp/src/vlib/main.c:1788#13 0x00007fd037ac1db7 in
vlib_worker_loop (vm=0x7fd00c58ac40) at
/development/libvpp/src/vlib/main.c:2008#14 0x00007fd037b19b1a in
vlib_worker_thread_fn* (arg=0x7fd00925e700) at
/development/libvpp/src/vlib/threads.c:1862
#15 0x00007fd037340c34 in clib_calljmp () at
/development/libvpp/src/vppinfra/longjmp.S:123
#16 0x00007fcf87ffed00 in ?? ()
#17 0x00007fd037b11cc3 in vlib_worker_thread_bootstrap_fn
(arg=0x7fd00925e700) at /development/libvpp/src/vlib/threads.c:585
Backtrace stopped: previous frame inner to this frame (corrupt stack?)
Thread 2 (Thread 0x7fcf8cbff700 (LWP 22777)):
#0 0x00007fd037ac81c6 in vlib_worker_thread_barrier_check () at
/development/libvpp/src/vlib/threads.h:485
#1 0x00007fd037ac259e in vlib_main_or_worker_loop (vm=0x7fd024530e40,
is_main=0) at /development/libvpp/src/vlib/main.c:1788
#2 0x00007fd037ac1db7 in vlib_worker_loop (vm=0x7fd024530e40) at
/development/libvpp/src/vlib/main.c:2008
#3 0x00007fd037b19b1a in vlib_worker_thread_fn (arg=0x7fd00925e600) at
/development/libvpp/src/vlib/threads.c:1862
#4 0x00007fd037340c34 in clib_calljmp () at
/development/libvpp/src/vppinfra/longjmp.S:123
#5 0x00007fcf8cbfed00 in ?? ()
#6 0x00007fd037b11cc3 in vlib_worker_thread_bootstrap_fn
(arg=0x7fd00925e600) at /development/libvpp/src/vlib/threads.c:585
Backtrace stopped: previous frame inner to this frame (corrupt stack?)
Thread 1 (Thread 0x7fd046adb400 (LWP 22729)):
#0 vlib_time_now (vm=0x7fd037d81c40 <vlib_global_main>) at
/development/libvpp/src/vlib/main.h:345
#1 0x00007fd037b1863c in vlib_worker_thread_barrier_release
(vm=0x7fd037d81c40 <vlib_global_main>) at
/development/libvpp/src/vlib/threads.c:1631
#2 0x00007fd039719985 in rtb_vpp_l2_xconnect_route_add_handle
(route_mapping=0x7fcff329d220) at
/development/libvpp/src/vpp/rtbrick/rtb_vpp_l2_xconnect.c:97
#3 0x00007fd03971a229 in rtb_vpp_l2_xconnect_route_handle
(route_mapping=0x7fcff329d220, action=0 '\000') at
/development/libvpp/src/vpp/rtbrick/rtb_vpp_l2_xconnect.c:179
#4 0x00007fd0396692a7 in rtb_vpp_route_mapping_process
(route_mapping=0x7fcff329d220, action=0 '\000') at
/development/libvpp/src/vpp/rtbrick/rtb_vpp_route.c:258
#5 0x00007fd03966cbd4 in rtb_vpp_adj_adjacency_route_handle
(adj_api_out=0x7fcff329d410, action=0 '\000') at
/development/libvpp/src/vpp/rtbrick/rtb_vpp_adj.c:247
#6 0x00007fd03966cd30 in rtb_vpp_adj_api_out_process
(table=0x56425216b120, object=0x56425cfa6b30, action=0 '\000') at
/development/libvpp/src/vpp/rtbrick/rtb_vpp_adj.c:283
#7 0x00007fd03966cdaf in rtb_vpp_adj_api_out_add_cb (table=0x56425216b120,
object=0x56425cfa6b30) at
/development/libvpp/src/vpp/rtbrick/rtb_vpp_adj.c:296
#8 0x00007fd044f98c2e in bds_handle_object_plugin
(table_handle=table_handle@entry=0x56425216b120,
index=index@entry=0x56425cfa6b30,
old_index=old_index@entry=0x0,
type=type@entry=BDS_PC_ADD) at
/development/rtbrick-infrastructure/code/bds/bds_core/src/bds_plugin.c:1020
#9 0x00007fd044f98e0b in bds_deferred_object_plugins
(obj_deleted=0x7fcff329d5a7 "", obj_handle=0x56425cfa6b30,
table_handle=0x56425216b120)
at
/development/rtbrick-infrastructure/code/bds/bds_core/src/bds_plugin.c:514
#10 bds_deferred_plugin_cb (table_handle=0x56425216b120,
obj_handle=0x56425cfa6b30, obj_deleted=obj_deleted@entry=0x7fcff329d5d7 "")
at
/development/rtbrick-infrastructure/code/bds/bds_core/src/bds_plugin.c:678
#11 0x00007fd044f9085d in bds_object_handle_plugin_publish
(table_handle=table_handle@entry=0x56425216b120, obj_handle=obj_handle@entry
=0x56425cfa6b30,
old_obj_handle=old_obj_handle@entry=0x0) at
/development/rtbrick-infrastructure/code/bds/bds_core/src/bds_object.c:3493
#12 0x00007fd044f90a47 in bds_object_handle_features (is_diff_index=0
'\000', add_vector=127, old_index=0x0, index=0x56425cfa6b30,
table=0x56425216b120)
at
/development/rtbrick-infrastructure/code/bds/bds_core/src/bds_object.c:3606
#13 bds_build_index (table=table@entry=0x56425216b120,
obj_map=obj_map@entry=0x56425cfa6960,
cp_offset=cp_offset@entry=0x7fcff329d6e0, cp_len=cp_len@entry
=0x7fcff329d8e0,
data_offset=data_offset@entry=80, add_bitmap=add_bitmap@entry=65535,
is_diff_index=<optimized out>) at
/development/rtbrick-infrastructure/code/bds/bds_core/src/bds_object.c:3942
#14 0x00007fd044f94bb7 in bds_object_add_map_and_get_index
(table_handle=0x56425216b120, obj_map=obj_map@entry=0x7fd03b6c2990
<gObjectMap+48>, obj_len=<optimized out>,
use_seq=use_seq@entry=0 '\000') at
/development/rtbrick-infrastructure/code/bds/bds_core/src/bds_object.c:5989
#15 0x00007fd044f95032 in bds_object_add_map (table_handle=<optimized out>,
obj_map=0x7fd03b6c2990 <gObjectMap+48>, obj_len=<optimized out>,
use_seq=use_seq@entry=0 '\000')
at
/development/rtbrick-infrastructure/code/bds/bds_core/src/bds_object.c:6137
#16 0x00007fd03a6690db in fib_adj_api_out_add
(fib_adjacency=fib_adjacency@entry=0x7fcff329e100) at
/development/libforwarding/pi/fib/code/src/fib_adjacency_api_out.c:862
#17 0x00007fd03a667819 in fib_adjacency_local_obj_process
(table=0x56425215d100, obj_handle=0x56425cfa5e10, action=action@entry=0
'\000')
at /development/libforwarding/pi/fib/code/src/fib_adjacency.c:123
#18 0x00007fd03a667831 in fib_adjacency_local_obj_add_cb
(tblHndl=<optimized out>, objHndl=<optimized out>) at
/development/libforwarding/pi/fib/code/src/fib_adjacency.c:136
#19 0x00007fd044f98c2e in bds_handle_object_plugin
(table_handle=table_handle@entry=0x56425215d100,
index=index@entry=0x56425cfa5e10,
old_index=old_index@entry=0x0,
type=type@entry=BDS_PC_ADD) at
/development/rtbrick-infrastructure/code/bds/bds_core/src/bds_plugin.c:1020
#20 0x00007fd044f98e0b in bds_deferred_object_plugins
(obj_deleted=0x7fcff329ed87 "", obj_handle=0x56425cfa5e10,
table_handle=0x56425215d100)
at
/development/rtbrick-infrastructure/code/bds/bds_core/src/bds_plugin.c:514
#21 bds_deferred_plugin_cb (table_handle=table_handle@entry=0x56425215d100,
obj_handle=0x56425cfa5e10, obj_deleted=obj_deleted@entry=0x7fcff329edd7 "")
at
/development/rtbrick-infrastructure/code/bds/bds_core/src/bds_plugin.c:678
#22 0x00007fd044f9920d in bds_deferred_plugin_cb_expiry
(table_handle=0x56425215d100) at
/development/rtbrick-infrastructure/code/bds/bds_core/src/bds_plugin.c:718
#23 0x00007fd0443a1656 in timer_dispatch (item=0x5642577bdb30, p=<optimized
out>) at /development/rtbrick-infrastructure/code/qb/lib/loop_timerlist.c:56
#24 0x00007fd04439efe7 in qb_loop_run_level (level=level@entry=0x56424fdda2c0)
at /development/rtbrick-infrastructure/code/qb/lib/loop.c:43
#25 0x00007fd04439f7d7 in qb_loop_run_vpp_wrapper (lp=0x56424fdda250,
wait_timer=wait_timer@entry=0) at
/development/rtbrick-infrastructure/code/qb/lib/loop.c:350
#26 0x00007fd0443ab1cf in lib_qb_service_start_event_wrapper_loop
(wait_timer=wait_timer@entry=0) at
/development/rtbrick-infrastructure/code/qb/lib/wrapper/lib_qb_service.c:290
#27 0x00007fd046672fa8 in bd_event_loop_run_once () at
/development/rtbrick-infrastructure/code/bd/src/bdinfra/bd_api.c:842
#28 0x00007fd0019354f8 in rtb_vpp_epoll_process (vm=0x7fd037d81c40
<vlib_global_main>, rt=0x7fd0245284c0, f=0x0) at
/development/libvpp/src/plugins/rtbrick/rtb_node.c:90
#29 0x00007fd037ac795d in vlib_process_bootstrap (_a=140531501042200) at
/development/libvpp/src/vlib/main.c:1464
#30 0x00007fd037340c34 in clib_calljmp () at
/development/libvpp/src/vppinfra/longjmp.S:123
#31 0x00007fd00a330a10 in ?? ()
#32 0x00007fd037ac739f in vlib_process_startup (vm=0x38fffffff8, p=0x48,
f=0x8) at /development/libvpp/src/vlib/main.c:1489
#33 0x0000000000000004 in ?? ()
#34 0x0000000000000001 in ?? ()
#35 0x0000000000000000 in ?? ()
(gdb)
Thanks,
Rajith
--
NOTICE TO
RECIPIENT This e-mail message and any attachments are
confidential and may be
privileged. If you received this e-mail in error,
any review, use,
dissemination, distribution, or copying of this e-mail is
strictly
prohibited. Please notify us immediately of the error by return
e-mail and
please delete this message from your system. For more
information about Rtbrick, please visit us at www.rtbrick.com
<http://www.rtbrick.com>
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#20732): https://lists.fd.io/g/vpp-dev/message/20732
Mute This Topic: https://lists.fd.io/mt/88481858/21656
Group Owner: vpp-dev+ow...@lists.fd.io
Unsubscribe: https://lists.fd.io/g/vpp-dev/unsub [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
