Hi All,
We have loaded our box with internet feed routes. Initially everything is
good.
But after *three hours* we observed a *vpp main thread crashed* due to
*segmentation
error *at mspace_put function.
#28 0x00007f0f802c0793 in unix_signal_handler (signum=11,
si=0x7f0f33c086b0, uc=0x7f0f33c08580)
at /development/libvpp/src/vlib/unix/main.c:127
#29 <signal handler called>
#30 __memset_avx2_erms () at
../sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S:145
#31 0x00007f0f7fa67735 in mspace_put (msp=0x130044010,
p_arg=0x130089a48) at /development/libvpp/src/vppinfra/dlmalloc.c:4316
We are using *20.09 version* and the complete* backtrace *is pasted
below. Any help in fixing the issue would be appreciated.
Thread 3 (Thread 0x7f0eccbfe700 (LWP 476)):
#0 0x00007f0f8023cff7 in vlib_worker_thread_barrier_check () at
/development/libvpp/src/vlib/threads.h:438
#1 0x00007f0f8023751e in vlib_main_or_worker_loop (vm=0x7f0f652dd300,
is_main=0) at /development/libvpp/src/vlib/main.c:1788
#2 0x00007f0f80236d37 in vlib_worker_loop (vm=0x7f0f652dd300) at
/development/libvpp/src/vlib/main.c:2008
#3 0x00007f0f8028e91a in vlib_worker_thread_fn (arg=0x7f0f4a974940)
at /development/libvpp/src/vlib/threads.c:1862
#4 0x00007f0f7fab5c34 in clib_calljmp () at
/development/libvpp/src/vppinfra/longjmp.S:123
#5 0x00007f0eccbfdec0 in ?? ()
[58/393]
#6 0x00007f0f80286ac3 in vlib_worker_thread_bootstrap_fn
(arg=0x7f0f4a974940) at /development/libvpp/src/vlib/threads.c:585
---Type <return> to continue, or q <return> to quit---
Backtrace stopped: previous frame inner to this frame (corrupt stack?)
Thread 2 (Thread 0x7f0ecd3ff700 (LWP 475)):
#0 0x00007f0f8023cfec in vlib_worker_thread_barrier_check () at
/development/libvpp/src/vlib/threads.h:438
#1 0x00007f0f8023751e in vlib_main_or_worker_loop (vm=0x7f0f64d99ec0,
is_main=0) at /development/libvpp/src/vlib/main.c:1788
#2 0x00007f0f80236d37 in vlib_worker_loop (vm=0x7f0f64d99ec0) at
/development/libvpp/src/vlib/main.c:2008
#3 0x00007f0f8028e91a in vlib_worker_thread_fn (arg=0x7f0f4a974840)
at /development/libvpp/src/vlib/threads.c:1862
#4 0x00007f0f7fab5c34 in clib_calljmp () at
/development/libvpp/src/vppinfra/longjmp.S:123
#5 0x00007f0ecd3feec0 in ?? ()
#6 0x00007f0f80286ac3 in vlib_worker_thread_bootstrap_fn
(arg=0x7f0f4a974840) at /development/libvpp/src/vlib/threads.c:585
Backtrace stopped: previous frame inner to this frame (corrupt stack?)
Thread 1 (Thread 0x7f0f8d076d00 (LWP 280)):
#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#1 0x00007f0f8c0d3921 in __GI_abort () at abort.c:79
#2 0x00007f0f81cbd253 in os_panic () at
/development/libvpp/src/vpp/vnet/main.c:572
#3 0x00007f0f7fa91aa9 in debugger () at
/development/libvpp/src/vppinfra/error.c:84
#4 0x00007f0f7fa91827 in _clib_error (how_to_die=2,
function_name=0x0, line_number=0, fmt=0x7f0f7fb613df "%s:%d (%s)
assertion `%s' fails")
at /development/libvpp/src/vppinfra/error.c:143
#5 0x00007f0f7fa98e61 in _vec_resize_inline (v=0x7f0f4a69fa90,
length_increment=16, data_bytes=16, header_bytes=0, data_align=1,
numa_id=255) at /development/libvpp/src/vppinfra/vec.h:154
#6 0x00007f0f7fa98b8b in va_format (s=0x7f0f4a69fa90 "",
fmt=0x7f0f802dbf1b "received signal %U, PC %U", va=0x7f0f33c065f0)
at /development/libvpp/src/vppinfra/format.c:403
#7 0x00007f0f7faa03c6 in format (s=0x7f0f4a69fa90 "",
fmt=0x7f0f802dbf1b "received signal %U, PC %U")
at /development/libvpp/src/vppinfra/format.c:428
#8 0x00007f0f802c0793 in unix_signal_handler (signum=6,
si=0x7f0f33c068b0, uc=0x7f0f33c06780)
at /development/libvpp/src/vlib/unix/main.c:127
#9 <signal handler called>
#10 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#11 0x00007f0f8c0d3921 in __GI_abort () at abort.c:79
#12 0x00007f0f81cbd253 in os_panic () at
/development/libvpp/src/vpp/vnet/main.c:572
#13 0x00007f0f7fa91aa9 in debugger () at
/development/libvpp/src/vppinfra/error.c:84
#14 0x00007f0f7fa91827 in _clib_error (how_to_die=2,
function_name=0x0, line_number=0, fmt=0x7f0f7fb613df "%s:%d (%s)
assertion `%s' f[23/393]
at /development/libvpp/src/vppinfra/error.c:143
---Type <return> to continue, or q <return> to quit---
#15 0x00007f0f7fa98e61 in _vec_resize_inline (v=0x7f0f4a69fa90,
length_increment=16, data_bytes=16, header_bytes=0, data_align=1,
numa_id=255) at /development/libvpp/src/vppinfra/vec.h:154
#16 0x00007f0f7fa98b8b in va_format (s=0x7f0f4a69fa90 "",
fmt=0x7f0f802dbf1b "received signal %U, PC %U", va=0x7f0f33c074f0)
at /development/libvpp/src/vppinfra/format.c:403
#17 0x00007f0f7faa03c6 in format (s=0x7f0f4a69fa90 "",
fmt=0x7f0f802dbf1b "received signal %U, PC %U")
at /development/libvpp/src/vppinfra/format.c:428
#18 0x00007f0f802c0793 in unix_signal_handler (signum=6,
si=0x7f0f33c077b0, uc=0x7f0f33c07680)
at /development/libvpp/src/vlib/unix/main.c:127
#19 <signal handler called>
#20 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#21 0x00007f0f8c0d3921 in __GI_abort () at abort.c:79
#22 0x00007f0f81cbd253 in os_panic () at
/development/libvpp/src/vpp/vnet/main.c:572
#23 0x00007f0f7fa91aa9 in debugger () at
/development/libvpp/src/vppinfra/error.c:84
#24 0x00007f0f7fa91827 in _clib_error (how_to_die=2,
function_name=0x0, line_number=0, fmt=0x7f0f7fb613df "%s:%d (%s)
assertion `%s' fails")
at /development/libvpp/src/vppinfra/error.c:143
#25 0x00007f0f7fa98e61 in _vec_resize_inline (v=0x7f0f4a69fa90,
length_increment=16, data_bytes=16, header_bytes=0, data_align=1,
numa_id=255) at /development/libvpp/src/vppinfra/vec.h:154
#26 0x00007f0f7fa98b8b in va_format (s=0x7f0f4a69fa90 "",
fmt=0x7f0f802dbf1b "received signal %U, PC %U", va=0x7f0f33c083f0)
at /development/libvpp/src/vppinfra/format.c:403
#27 0x00007f0f7faa03c6 in format (s=0x7f0f4a69fa90 "",
fmt=0x7f0f802dbf1b "received signal %U, PC %U")
at /development/libvpp/src/vppinfra/format.c:428
#28 0x00007f0f802c0793 in unix_signal_handler (signum=11,
si=0x7f0f33c086b0, uc=0x7f0f33c08580)
at /development/libvpp/src/vlib/unix/main.c:127
#29 <signal handler called>
#30 __memset_avx2_erms () at
../sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S:145
#31 0x00007f0f7fa67735 in mspace_put (msp=0x130044010,
p_arg=0x130089a48) at /development/libvpp/src/vppinfra/dlmalloc.c:4316
#32 0x00007f0f81a4e0ab in clib_mem_free (p=0x130089a48) at
/development/libvpp/src/vppinfra/mem.h:224
#33 0x00007f0f81a4dff2 in vl_msg_api_free_w_region
(vlib_rp=0x130023000, a=0x130089a58)
at /development/libvpp/src/vlibmemory/memory_shared.c:306
#34 0x00007f0f81a4e0e9 in vl_msg_api_free (a=0x130089a58) at
/development/libvpp/src/vlibmemory/memory_shared.c:314
#35 0x00007f0f81a8f22b in vl_msg_api_handler_with_vm_node
(am=0x7f0f81ca2d18 <api_global_main>, vlib_rp=0x130023000,
the_msg=0x130089a58,
vm=0x7f0f804f6c40 <vlib_global_main>, node=0x7f0f4aa186c0,
is_private=0 '\000') at
/development/libvpp/src/vlibapi/api_shared.c:657
#36 0x00007f0f81a4ac3b in vl_mem_api_handle_rpc (vm=0x7f0f804f6c40
<vlib_global_main>, node=0x7f0f4aa186c0)
at /development/libvpp/src/vlibmemory/memory_api.c:746
---Type <return> to continue, or q <return> to quit---
#37 0x00007f0f81a6c787 in vl_api_clnt_process (vm=0x7f0f804f6c40
<vlib_global_main>, node=0x7f0f4aa186c0, f=0x0)
at /development/libvpp/src/vlibmemory/vlib_api.c:337
#38 0x00007f0f8023c8dd in vlib_process_bootstrap (_a=139704040261608)
at /development/libvpp/src/vlib/main.c:1464
#39 0x00007f0f7fab5c34 in clib_calljmp () at
/development/libvpp/src/vppinfra/longjmp.S:123
#40 0x00007f0f61b187e0 in ?? ()
#41 0x00007f0f8023c31f in vlib_process_startup (vm=0x11061b18850,
p=0x42b9240c68966, f=0x110) at
/development/libvpp/src/vlib/main.c:1489
#42 0x00000033660c30c0 in ?? ()
#43 0x000000000000001c in ?? ()
#44 0x0000003700000001 in ?? ()
#45 0x000000000000001e in ?? ()
#46 0x00007f0f804f6c40 in ?? () from /usr/local/lib/libvlib.so.1.0.1
#47 0x00007f0f61b18950 in ?? ()
#48 0x00007f0f81a489d1 in memclnt_queue_callback (vm=<error reading
variable: Cannot access memory at address 0xfffffffffffffff8>)
at /development/libvpp/src/vlibmemory/memory_api.c:110
Backtrace stopped: previous frame inner to this frame (corrupt stack?)
Thanks and Regards,
Sudhir
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#19018): https://lists.fd.io/g/vpp-dev/message/19018
Mute This Topic: https://lists.fd.io/mt/81600282/21656
Group Owner: vpp-dev+ow...@lists.fd.io
Unsubscribe: https://lists.fd.io/g/vpp-dev/unsub [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
