GitHub user bryancall opened an issue:
https://github.com/apache/trafficserver/issues/1532
ATS 7.1 release running out of memory
Here is a backtrace from a bunch of cores I am getting from 7.1. They are
mostly issues with memory allocation:
[bcall@e24 ~]$ egrep '\#2 |\#3 |\#4 |\#5 ' ~/gdb.out
> #2 0x00000000007857e8 in UnixNetProcessor::allocate_vc (this=0x105cc20,
t=0x0) at ../../../../trafficserver/iocore/net/UnixNetProcessor.cc:460
> #3 0x0000000000782bdf in NetAccept::do_blocking_accept
(this=0x2b9b280253d0, t=0x2b9b281d6050) at
../../../../trafficserver/iocore/net/UnixNetAccept.cc:259
> #4 0x0000000000783a1b in NetAccept::acceptLoopEvent
(this=0x2b9b280253d0, event=1, e=0x2b9b281cf3c0) at
../../../../trafficserver/iocore/net/UnixNetAccept.cc:469
> #5 0x0000000000515468 in Continuation::handleEvent (this=0x2b9b280253d0,
event=1, data=0x2b9b281cf3c0) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/I_Continuation.h:153
> #2 0x00002af5d64098d0 in ink_abort (message_format=0x2af5d64200f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002af5d640deff in ats_memalign (alignment=4096, size=4194304) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x0000000000515638 in IOBufferData::alloc (this=0x2af631221260,
size_index=-4194304, type=MEMALIGNED) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/P_IOBuffer.h:288
> #5 0x0000000000515579 in new_IOBufferData_internal (loc=0x814bf0
"memory/IOBuffer/../../../../trafficserver/iocore/cache/Cache.cc:2498",
size_index=-4194304, type=MEMALIGNED) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/P_IOBuffer.h:264
> #2 0x00002b46476178d0 in ink_abort (message_format=0x2b464762e0f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002b464761beff in ats_memalign (alignment=4096, size=33554432) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x00002b464761c938 in freelist_new (f=0x14ec000) at
../../../../trafficserver/lib/ts/ink_queue.cc:212
> #5 0x00002b464761c832 in ink_freelist_new (f=0x14ec000) at
../../../../trafficserver/lib/ts/ink_queue.cc:183
> #2 0x00002aed89fc08d0 in ink_abort (message_format=0x2aed89fd70f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002aed89fc4eff in ats_memalign (alignment=4096, size=987136) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x00002aed89fc5938 in freelist_new (f=0x25bb5d0) at
../../../../trafficserver/lib/ts/ink_queue.cc:212
> #5 0x00002aed89fc5832 in ink_freelist_new (f=0x25bb5d0) at
../../../../trafficserver/lib/ts/ink_queue.cc:183
> #2 0x00002b3d66d4a8d0 in ink_abort (message_format=0x2b3d66d610f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002b3d66d4eeff in ats_memalign (alignment=4096, size=33554432) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x00002b3d66d4f938 in freelist_new (f=0x1f50000) at
../../../../trafficserver/lib/ts/ink_queue.cc:212
> #5 0x00002b3d66d4f832 in ink_freelist_new (f=0x1f50000) at
../../../../trafficserver/lib/ts/ink_queue.cc:183
> #2 0x00002af391be38d0 in ink_abort (message_format=0x2af391bfa0f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002af391be7eff in ats_memalign (alignment=4096, size=3407872) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x0000000000515638 in IOBufferData::alloc (this=0x2ab464602e90,
size_index=-3407872, type=MEMALIGNED) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/P_IOBuffer.h:288
> #5 0x0000000000515579 in new_IOBufferData_internal (loc=0x814bf0
"memory/IOBuffer/../../../../trafficserver/iocore/cache/Cache.cc:2498",
size_index=-3407872, type=MEMALIGNED) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/P_IOBuffer.h:264
> #2 0x0000000000534d3d in TSHttpSsnClientAddrGet (ssnp=0x2ab21f4d1bc0) at
../../../trafficserver/proxy/InkAPI.cc:5445
> #3 0x0000000000534d8b in TSHttpTxnClientAddrGet (txnp=0x2ab48b84cc00) at
../../../trafficserver/proxy/InkAPI.cc:5453
> #4 0x00002aaab248f8f6 in http_hook (contp=0x275cf40, event=60006,
edata=0x2ab48b84cc00) at INKPluginInit.cc:174
> #5 0x000000000052a025 in INKContInternal::handle_event (this=0x275cf40,
event=60006, edata=0x2ab48b84cc00) at
../../../trafficserver/proxy/InkAPI.cc:1048
> #2 0x00002ab0ec4f18d0 in ink_abort (message_format=0x2ab0ec5074f0
"%s:%d: failed assertion `%s`") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002ab0ec4eefd4 in _ink_assert (expression=0x824641 "0",
file=0x824568 "../../../../trafficserver/iocore/net/UnixNetVConnection.cc",
line=188) at ../../../../trafficserver/lib/ts/ink_assert.cc:37
> #4 0x0000000000786ff6 in write_signal_and_update (event=101,
vc=0x2aac6cd09650) at
../../../../trafficserver/iocore/net/UnixNetVConnection.cc:188
> #5 0x000000000078811d in write_to_net_io (nh=0x2ab0f4f12e60,
vc=0x2aac6cd09650, thread=0x2ab0f4f0f010) at
../../../../trafficserver/iocore/net/UnixNetVConnection.cc:528
> #2 0x00002b0ac8ecd8d0 in ink_abort (message_format=0x2b0ac8ee40f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002b0ac8ed1eff in ats_memalign (alignment=4096, size=4194304) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x0000000000515638 in IOBufferData::alloc (this=0x2b0b4952d020,
size_index=-4194304, type=MEMALIGNED) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/P_IOBuffer.h:288
> #5 0x0000000000515579 in new_IOBufferData_internal (loc=0x814bf0
"memory/IOBuffer/../../../../trafficserver/iocore/cache/Cache.cc:2498",
size_index=-4194304, type=MEMALIGNED) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/P_IOBuffer.h:264
> #2 0x00002ade6dcf08d0 in ink_abort (message_format=0x2ade6dd070f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002ade6dcf4eff in ats_memalign (alignment=4096, size=1048576) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x00002ade6dcf5938 in freelist_new (f=0x22fa000) at
../../../../trafficserver/lib/ts/ink_queue.cc:212
> #5 0x00002ade6dcf5832 in ink_freelist_new (f=0x22fa000) at
../../../../trafficserver/lib/ts/ink_queue.cc:183
> #2 0x00002b65ae07d8d0 in ink_abort (message_format=0x2b65ae0940f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002b65ae081eff in ats_memalign (alignment=4096, size=1048576) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x00002b65ae082938 in freelist_new (f=0x1a98000) at
../../../../trafficserver/lib/ts/ink_queue.cc:212
> #5 0x00002b65ae082832 in ink_freelist_new (f=0x1a98000) at
../../../../trafficserver/lib/ts/ink_queue.cc:183
> #2 0x00000000005d7eb8 in
Http1ClientTransaction::cancel_inactivity_timeout (this=0x2b3700508a10) at
../../../../trafficserver/proxy/http/Http1ClientTransaction.h:162
> #3 0x00000000005e8166 in HttpSM::state_read_server_response_header
(this=0x2b3761188160, event=100, data=0x2aac688b5028) at
../../../../trafficserver/proxy/http/HttpSM.cc:1861
> #4 0x00000000005eb09b in HttpSM::main_handler (this=0x2b3761188160,
event=100, data=0x2aac688b5028) at
../../../../trafficserver/proxy/http/HttpSM.cc:2663
> #5 0x0000000000515468 in Continuation::handleEvent (this=0x2b3761188160,
event=100, data=0x2aac688b5028) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/I_Continuation.h:153
> #2 0x00002b38e7893aa6 in backtrace () from /lib64/libc.so.6
> #3 0x00002b38e531c1ae in ink_stack_trace_dump () at
../../../../trafficserver/lib/ts/ink_stack_trace.cc:61
> #4 0x00002b38e531e4a2 in signal_crash_handler (signo=11) at
../../../../trafficserver/lib/ts/signals.cc:186
> #5 0x0000000000512750 in crash_logger_invoke (signo=11,
info=0x2b38f0503230, ctx=0x2b38f0503100) at
../../../trafficserver/proxy/Crash.cc:169
> #2 0x00002b13bf3db8d0 in ink_abort (message_format=0x2b13bf3f20f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002b13bf3dfeff in ats_memalign (alignment=4096, size=1048576) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x00002b13bf3e0938 in freelist_new (f=0x1162000) at
../../../../trafficserver/lib/ts/ink_queue.cc:212
> #5 0x00002b13bf3e0832 in ink_freelist_new (f=0x1162000) at
../../../../trafficserver/lib/ts/ink_queue.cc:183
> #2 0x00002aea86f4a8d0 in ink_abort (message_format=0x2aea86f610f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002aea86f4eeff in ats_memalign (alignment=4096, size=4194304) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x0000000000515638 in IOBufferData::alloc (this=0x2aeac67764d0,
size_index=-4194304, type=MEMALIGNED) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/P_IOBuffer.h:288
> #5 0x0000000000515579 in new_IOBufferData_internal (loc=0x814bf0
"memory/IOBuffer/../../../../trafficserver/iocore/cache/Cache.cc:2498",
size_index=-4194304, type=MEMALIGNED) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/P_IOBuffer.h:264
> #2 0x00002b456ad1f8d0 in ink_abort (message_format=0x2b456ad360f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002b456ad23eff in ats_memalign (alignment=4096, size=1048576) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x00002b456ad24938 in freelist_new (f=0x11f4000) at
../../../../trafficserver/lib/ts/ink_queue.cc:212
> #5 0x00002b456ad24832 in ink_freelist_new (f=0x11f4000) at
../../../../trafficserver/lib/ts/ink_queue.cc:183
> #2 0x00002ad23c1bd8d0 in ink_abort (message_format=0x2ad23c1d40f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002ad23c1c1eff in ats_memalign (alignment=4096, size=3670016) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x0000000000515638 in IOBufferData::alloc (this=0x2ab1e5d80c50,
size_index=-3670016, type=MEMALIGNED) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/P_IOBuffer.h:288
> #5 0x0000000000515579 in new_IOBufferData_internal (loc=0x814bf0
"memory/IOBuffer/../../../../trafficserver/iocore/cache/Cache.cc:2498",
size_index=-3670016, type=MEMALIGNED) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/P_IOBuffer.h:264
> #2 0x00002b5947fa88d0 in ink_abort (message_format=0x2b5947fbf0f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002b5947faceff in ats_memalign (alignment=4096, size=1048576) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x00002b5947fad938 in freelist_new (f=0x277c000) at
../../../../trafficserver/lib/ts/ink_queue.cc:212
> #5 0x00002b5947fad832 in ink_freelist_new (f=0x277c000) at
../../../../trafficserver/lib/ts/ink_queue.cc:183
> #2 0x00002ae0db1c58d0 in ink_abort (message_format=0x2ae0db1dc0f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002ae0db1c9eff in ats_memalign (alignment=4096, size=2883584) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x0000000000515638 in IOBufferData::alloc (this=0x2ab4f572a680,
size_index=-2883584, type=MEMALIGNED) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/P_IOBuffer.h:288
> #5 0x0000000000515579 in new_IOBufferData_internal (loc=0x814bf0
"memory/IOBuffer/../../../../trafficserver/iocore/cache/Cache.cc:2498",
size_index=-2883584, type=MEMALIGNED) at
/home/bcall/dev/yahoo/build/_build/ats_build/../../trafficserver/iocore/eventsystem/P_IOBuffer.h:264
> #2 0x00000000005d7e7b in Http1ClientTransaction::set_inactivity_timeout
(this=0x2b419810b910, timeout_in=30000000000) at
../../../../trafficserver/proxy/http/Http1ClientTransaction.h:156
> #3 0x00000000005f642f in HttpSM::do_setup_post_tunnel
(this=0x2ab4abe81d80, to_vc_type=HTTP_SERVER_VC) at
../../../../trafficserver/proxy/http/HttpSM.cc:5719
> #4 0x00000000005e8866 in HttpSM::state_send_server_request_header
(this=0x2ab4abe81d80, event=103, data=0x2aac6c2e5520) at
../../../../trafficserver/proxy/http/HttpSM.cc:2002
> #5 0x00000000005eb09b in HttpSM::main_handler (this=0x2ab4abe81d80,
event=103, data=0x2aac6c2e5520) at
../../../../trafficserver/proxy/http/HttpSM.cc:2663
> #2 0x00002af1d09478d0 in ink_abort (message_format=0x2af1d095e0f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002af1d094beff in ats_memalign (alignment=4096, size=67108864) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x00002af1d094c938 in freelist_new (f=0x1714000) at
../../../../trafficserver/lib/ts/ink_queue.cc:212
> #5 0x00002af1d094c832 in ink_freelist_new (f=0x1714000) at
../../../../trafficserver/lib/ts/ink_queue.cc:183
> #2 0x00002b461e6fd8d0 in ink_abort (message_format=0x2b461e7140f0
"couldn't allocate %zu bytes at alignment %zu - insufficient memory") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002b461e701eff in ats_memalign (alignment=4096, size=1048576) at
../../../../trafficserver/lib/ts/ink_memory.cc:108
> #4 0x00002b461e702938 in freelist_new (f=0x17e4000) at
../../../../trafficserver/lib/ts/ink_queue.cc:212
> #5 0x00002b461e702832 in ink_freelist_new (f=0x17e4000) at
../../../../trafficserver/lib/ts/ink_queue.cc:183
> #2 0x0000000000000000 in ?? ()
> #2 0x00002b4111ee08d0 in ink_abort (message_format=0x2b4111ef7038
"couldn't allocate %zu bytes") at
../../../../trafficserver/lib/ts/ink_error.cc:99
> #3 0x00002b4111ee4db6 in ats_malloc (size=106496) at
../../../../trafficserver/lib/ts/ink_memory.cc:60
> #4 0x0000000000690bd0 in LogFile::write_ascii_logbuffer3
(this=0x2d51e70, buffer_header=0x2ab4bb11c800, alt_format=0x0) at
../../../../trafficserver/proxy/logging/LogFile.cc:457
> #5 0x00000000006906ca in LogFile::preproc_and_try_delete
(this=0x2d51e70, lb=0x2b41a570cbb0) at
../../../../trafficserver/proxy/logging/LogFile.cc:331
> #2 0x0000000000000000 in ?? ()
----
----
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
