Hi All, The segmentation fault is happening at memset in the code below.
#if CLIB_DEBUG <http://rajith/lxr/http/ident?sn=vpp-19-08;i=CLIB_DEBUG> > 0 && !defined(CLIB_SANITIZE_ADDR) */* Poison the object */* { size_t psize = mspace_usable_size <http://rajith/lxr/http/ident?sn=vpp-19-08;i=mspace_usable_size> (object_header); memset (object_header, 0x13, psize); } #endif Not sure how to proceed further to root cause the issue. Thanks, Sudhir On Thu, Mar 25, 2021 at 5:31 PM Sudhir CR <sud...@rtbrick.com> wrote: > Hi All, > We have loaded our box with internet feed routes. Initially everything is > good. > But after *three hours* we observed a *vpp main thread crashed* due to > *segmentation > error *at mspace_put function. > > #28 0x00007f0f802c0793 in unix_signal_handler (signum=11, si=0x7f0f33c086b0, > uc=0x7f0f33c08580) > at /development/libvpp/src/vlib/unix/main.c:127 > #29 <signal handler called> > #30 __memset_avx2_erms () at > ../sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S:145 > #31 0x00007f0f7fa67735 in mspace_put (msp=0x130044010, p_arg=0x130089a48) at > /development/libvpp/src/vppinfra/dlmalloc.c:4316 > > We are using *20.09 version* and the complete* backtrace *is pasted below. > Any help in fixing the issue would be appreciated. > > Thread 3 (Thread 0x7f0eccbfe700 (LWP 476)): > #0 0x00007f0f8023cff7 in vlib_worker_thread_barrier_check () at > /development/libvpp/src/vlib/threads.h:438 > #1 0x00007f0f8023751e in vlib_main_or_worker_loop (vm=0x7f0f652dd300, > is_main=0) at /development/libvpp/src/vlib/main.c:1788 > #2 0x00007f0f80236d37 in vlib_worker_loop (vm=0x7f0f652dd300) at > /development/libvpp/src/vlib/main.c:2008 > #3 0x00007f0f8028e91a in vlib_worker_thread_fn (arg=0x7f0f4a974940) at > /development/libvpp/src/vlib/threads.c:1862 > #4 0x00007f0f7fab5c34 in clib_calljmp () at > /development/libvpp/src/vppinfra/longjmp.S:123 > #5 0x00007f0eccbfdec0 in ?? () > [58/393] > #6 0x00007f0f80286ac3 in vlib_worker_thread_bootstrap_fn > (arg=0x7f0f4a974940) at /development/libvpp/src/vlib/threads.c:585 > ---Type <return> to continue, or q <return> to quit--- > Backtrace stopped: previous frame inner to this frame (corrupt stack?) > > Thread 2 (Thread 0x7f0ecd3ff700 (LWP 475)): > #0 0x00007f0f8023cfec in vlib_worker_thread_barrier_check () at > /development/libvpp/src/vlib/threads.h:438 > #1 0x00007f0f8023751e in vlib_main_or_worker_loop (vm=0x7f0f64d99ec0, > is_main=0) at /development/libvpp/src/vlib/main.c:1788 > #2 0x00007f0f80236d37 in vlib_worker_loop (vm=0x7f0f64d99ec0) at > /development/libvpp/src/vlib/main.c:2008 > #3 0x00007f0f8028e91a in vlib_worker_thread_fn (arg=0x7f0f4a974840) at > /development/libvpp/src/vlib/threads.c:1862 > #4 0x00007f0f7fab5c34 in clib_calljmp () at > /development/libvpp/src/vppinfra/longjmp.S:123 > #5 0x00007f0ecd3feec0 in ?? () > #6 0x00007f0f80286ac3 in vlib_worker_thread_bootstrap_fn > (arg=0x7f0f4a974840) at /development/libvpp/src/vlib/threads.c:585 > Backtrace stopped: previous frame inner to this frame (corrupt stack?) > > Thread 1 (Thread 0x7f0f8d076d00 (LWP 280)): > #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 > #1 0x00007f0f8c0d3921 in __GI_abort () at abort.c:79 > #2 0x00007f0f81cbd253 in os_panic () at > /development/libvpp/src/vpp/vnet/main.c:572 > #3 0x00007f0f7fa91aa9 in debugger () at > /development/libvpp/src/vppinfra/error.c:84 > #4 0x00007f0f7fa91827 in _clib_error (how_to_die=2, function_name=0x0, > line_number=0, fmt=0x7f0f7fb613df "%s:%d (%s) assertion `%s' fails") > at /development/libvpp/src/vppinfra/error.c:143 > #5 0x00007f0f7fa98e61 in _vec_resize_inline (v=0x7f0f4a69fa90, > length_increment=16, data_bytes=16, header_bytes=0, data_align=1, > numa_id=255) at /development/libvpp/src/vppinfra/vec.h:154 > #6 0x00007f0f7fa98b8b in va_format (s=0x7f0f4a69fa90 "", fmt=0x7f0f802dbf1b > "received signal %U, PC %U", va=0x7f0f33c065f0) > at /development/libvpp/src/vppinfra/format.c:403 > #7 0x00007f0f7faa03c6 in format (s=0x7f0f4a69fa90 "", fmt=0x7f0f802dbf1b > "received signal %U, PC %U") > at /development/libvpp/src/vppinfra/format.c:428 > #8 0x00007f0f802c0793 in unix_signal_handler (signum=6, si=0x7f0f33c068b0, > uc=0x7f0f33c06780) > at /development/libvpp/src/vlib/unix/main.c:127 > #9 <signal handler called> > #10 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 > #11 0x00007f0f8c0d3921 in __GI_abort () at abort.c:79 > #12 0x00007f0f81cbd253 in os_panic () at > /development/libvpp/src/vpp/vnet/main.c:572 > #13 0x00007f0f7fa91aa9 in debugger () at > /development/libvpp/src/vppinfra/error.c:84 > #14 0x00007f0f7fa91827 in _clib_error (how_to_die=2, function_name=0x0, > line_number=0, fmt=0x7f0f7fb613df "%s:%d (%s) assertion `%s' f[23/393] > at /development/libvpp/src/vppinfra/error.c:143 > ---Type <return> to continue, or q <return> to quit--- > #15 0x00007f0f7fa98e61 in _vec_resize_inline (v=0x7f0f4a69fa90, > length_increment=16, data_bytes=16, header_bytes=0, data_align=1, > numa_id=255) at /development/libvpp/src/vppinfra/vec.h:154 > #16 0x00007f0f7fa98b8b in va_format (s=0x7f0f4a69fa90 "", fmt=0x7f0f802dbf1b > "received signal %U, PC %U", va=0x7f0f33c074f0) > at /development/libvpp/src/vppinfra/format.c:403 > #17 0x00007f0f7faa03c6 in format (s=0x7f0f4a69fa90 "", fmt=0x7f0f802dbf1b > "received signal %U, PC %U") > at /development/libvpp/src/vppinfra/format.c:428 > #18 0x00007f0f802c0793 in unix_signal_handler (signum=6, si=0x7f0f33c077b0, > uc=0x7f0f33c07680) > at /development/libvpp/src/vlib/unix/main.c:127 > #19 <signal handler called> > #20 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 > #21 0x00007f0f8c0d3921 in __GI_abort () at abort.c:79 > #22 0x00007f0f81cbd253 in os_panic () at > /development/libvpp/src/vpp/vnet/main.c:572 > #23 0x00007f0f7fa91aa9 in debugger () at > /development/libvpp/src/vppinfra/error.c:84 > #24 0x00007f0f7fa91827 in _clib_error (how_to_die=2, function_name=0x0, > line_number=0, fmt=0x7f0f7fb613df "%s:%d (%s) assertion `%s' fails") > at /development/libvpp/src/vppinfra/error.c:143 > #25 0x00007f0f7fa98e61 in _vec_resize_inline (v=0x7f0f4a69fa90, > length_increment=16, data_bytes=16, header_bytes=0, data_align=1, > numa_id=255) at /development/libvpp/src/vppinfra/vec.h:154 > #26 0x00007f0f7fa98b8b in va_format (s=0x7f0f4a69fa90 "", fmt=0x7f0f802dbf1b > "received signal %U, PC %U", va=0x7f0f33c083f0) > at /development/libvpp/src/vppinfra/format.c:403 > #27 0x00007f0f7faa03c6 in format (s=0x7f0f4a69fa90 "", fmt=0x7f0f802dbf1b > "received signal %U, PC %U") > at /development/libvpp/src/vppinfra/format.c:428 > #28 0x00007f0f802c0793 in unix_signal_handler (signum=11, si=0x7f0f33c086b0, > uc=0x7f0f33c08580) > at /development/libvpp/src/vlib/unix/main.c:127 > #29 <signal handler called> > #30 __memset_avx2_erms () at > ../sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S:145 > #31 0x00007f0f7fa67735 in mspace_put (msp=0x130044010, p_arg=0x130089a48) at > /development/libvpp/src/vppinfra/dlmalloc.c:4316 > #32 0x00007f0f81a4e0ab in clib_mem_free (p=0x130089a48) at > /development/libvpp/src/vppinfra/mem.h:224 > #33 0x00007f0f81a4dff2 in vl_msg_api_free_w_region (vlib_rp=0x130023000, > a=0x130089a58) > at /development/libvpp/src/vlibmemory/memory_shared.c:306 > #34 0x00007f0f81a4e0e9 in vl_msg_api_free (a=0x130089a58) at > /development/libvpp/src/vlibmemory/memory_shared.c:314 > #35 0x00007f0f81a8f22b in vl_msg_api_handler_with_vm_node (am=0x7f0f81ca2d18 > <api_global_main>, vlib_rp=0x130023000, the_msg=0x130089a58, > vm=0x7f0f804f6c40 <vlib_global_main>, node=0x7f0f4aa186c0, is_private=0 > '\000') at /development/libvpp/src/vlibapi/api_shared.c:657 > #36 0x00007f0f81a4ac3b in vl_mem_api_handle_rpc (vm=0x7f0f804f6c40 > <vlib_global_main>, node=0x7f0f4aa186c0) > at /development/libvpp/src/vlibmemory/memory_api.c:746 > ---Type <return> to continue, or q <return> to quit--- > #37 0x00007f0f81a6c787 in vl_api_clnt_process (vm=0x7f0f804f6c40 > <vlib_global_main>, node=0x7f0f4aa186c0, f=0x0) > at /development/libvpp/src/vlibmemory/vlib_api.c:337 > #38 0x00007f0f8023c8dd in vlib_process_bootstrap (_a=139704040261608) at > /development/libvpp/src/vlib/main.c:1464 > #39 0x00007f0f7fab5c34 in clib_calljmp () at > /development/libvpp/src/vppinfra/longjmp.S:123 > #40 0x00007f0f61b187e0 in ?? () > #41 0x00007f0f8023c31f in vlib_process_startup (vm=0x11061b18850, > p=0x42b9240c68966, f=0x110) at /development/libvpp/src/vlib/main.c:1489 > #42 0x00000033660c30c0 in ?? () > #43 0x000000000000001c in ?? () > #44 0x0000003700000001 in ?? () > #45 0x000000000000001e in ?? () > #46 0x00007f0f804f6c40 in ?? () from /usr/local/lib/libvlib.so.1.0.1 > #47 0x00007f0f61b18950 in ?? () > #48 0x00007f0f81a489d1 in memclnt_queue_callback (vm=<error reading variable: > Cannot access memory at address 0xfffffffffffffff8>) > at /development/libvpp/src/vlibmemory/memory_api.c:110 > Backtrace stopped: previous frame inner to this frame (corrupt stack?) > > Thanks and Regards, > > Sudhir > > >
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#19032): https://lists.fd.io/g/vpp-dev/message/19032 Mute This Topic: https://lists.fd.io/mt/81600282/21656 Group Owner: vpp-dev+ow...@lists.fd.io Unsubscribe: https://lists.fd.io/g/vpp-dev/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
