Hello, Using haproxy v1.9.8 + 3d4052131ba3abe63d9cde213d6b89763f5bf828 we are seeing unexplained behavior after a reload where ~5/7 cores at 100% are used over the 16 possible in the config. It's hard to know whether it is still handling traffic since the unix socket is not reachable anymore. However the few connections attached to this process sometimes send one request every few minutes. I wonder whether, using the thread traces, it is an expected behavior as it seems some threads are stuck in an infinite loop.
we are using: nbthread 16 pool-purge-delay 10s Could you advise? 0x000055f59f3bdf40 in task_wakeup (f=512, t=0x55f5dc2ad910) at include/proto/task.h:155 155 state = HA_ATOMIC_OR(&t->state, f); (gdb) thread apply all bt Thread 16 (Thread 0x7f79e0b71700 (LWP 17360)): #0 0x00007ffd9ac747aa in clock_gettime () #1 0x00007f79e5e87c6d in clock_gettime () from /lib64/libc.so.6 #2 0x000055f59f2ac96c in now_cpu_time () at include/common/time.h:538 #3 activity_count_runtime () at include/proto/activity.h:52 #4 _do_poll (p=<optimized out>, exp=1885071588) at src/ev_epoll.c:152 #5 0x000055f59f335323 in run_poll_loop () at src/haproxy.c:2704 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #7 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 15 (Thread 0x7f79e0370700 (LWP 17361)): #0 0x00007ffd9ac747aa in clock_gettime () #1 0x00007f79e5e87c6d in clock_gettime () from /lib64/libc.so.6 #2 0x000055f59f2ac96c in now_cpu_time () at include/common/time.h:538 #3 activity_count_runtime () at include/proto/activity.h:52 #4 _do_poll (p=<optimized out>, exp=1885071590) at src/ev_epoll.c:152 #5 0x000055f59f335323 in run_poll_loop () at src/haproxy.c:2704 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #7 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 14 (Thread 0x7f79dfb6f700 (LWP 17362)): #0 0x00007f79e5e72483 in epoll_wait () from /lib64/libc.so.6 #1 0x000055f59f2acad8 in _do_poll (p=<optimized out>, exp=1885071590) at src/ev_epoll.c:156 #2 0x000055f59f335323 in run_poll_loop () at src/haproxy.c:2704 #3 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #4 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 13 (Thread 0x7f79df36e700 (LWP 17363)): #0 0x00007f79e5e72483 in epoll_wait () from /lib64/libc.so.6 #1 0x000055f59f2acad8 in _do_poll (p=<optimized out>, exp=1885071590) at src/ev_epoll.c:156 #2 0x000055f59f335323 in run_poll_loop () at src/haproxy.c:2704 #3 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #4 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 12 (Thread 0x7f79deb6d700 (LWP 17364)): #0 __task_unlink_rq (t=0x7f79cccacf60) at include/proto/task.h:212 #1 process_runnable_tasks () at src/task.c:329 #2 0x000055f59f33534f in run_poll_loop () at src/haproxy.c:2667 #3 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #4 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 11 (Thread 0x7f79de36c700 (LWP 17365)): #0 task_wakeup (f=512, t=0x7f79c04e80c0) at include/proto/task.h:155 #1 wake_expired_tasks () at src/task.c:209 #2 0x000055f59f335372 in run_poll_loop () at src/haproxy.c:2675 #3 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #4 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 10 (Thread 0x7f79ddb6b700 (LWP 17366)): #0 0x000055f59f3bdf02 in wake_expired_tasks () at src/task.c:214 #1 0x000055f59f335372 in run_poll_loop () at src/haproxy.c:2675 #2 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #3 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 9 (Thread 0x7f79dd36a700 (LWP 17367)): #0 0x00007ffd9ac747aa in clock_gettime () #1 0x00007f79e5e87c6d in clock_gettime () from /lib64/libc.so.6 #2 0x000055f59f2acba6 in now_cpu_time () at include/common/time.h:538 #3 tv_leaving_poll (timeout=<optimized out>, interrupted=0) at include/common/time.h:590 #4 _do_poll (p=<optimized out>, exp=1885071586) at src/ev_epoll.c:169 #5 0x000055f59f335323 in run_poll_loop () at src/haproxy.c:2704 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #7 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 8 (Thread 0x7f79dcb69700 (LWP 17368)): #0 0x000055f59f3bdf02 in wake_expired_tasks () at src/task.c:214 #1 0x000055f59f335372 in run_poll_loop () at src/haproxy.c:2675 #2 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #3 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 7 (Thread 0x7f79bffff700 (LWP 17369)): #0 process_runnable_tasks () at src/task.c:374 #1 0x000055f59f33534f in run_poll_loop () at src/haproxy.c:2667 #2 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #3 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 6 (Thread 0x7f79bf7fe700 (LWP 17370)): #0 0x000055f59f3bdf02 in wake_expired_tasks () at src/task.c:214 #1 0x000055f59f335372 in run_poll_loop () at src/haproxy.c:2675 #2 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #3 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x7f79beffd700 (LWP 17371)): #0 0x000055f59f3bdf02 in wake_expired_tasks () at src/task.c:214 #1 0x000055f59f335372 in run_poll_loop () at src/haproxy.c:2675 #2 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #3 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x7f79be7fc700 (LWP 17372)): #0 __task_unlink_rq (t=0x7f79a066fd00) at include/proto/task.h:212 #1 process_runnable_tasks () at src/task.c:329 #2 0x000055f59f33534f in run_poll_loop () at src/haproxy.c:2667 #3 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #4 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 ---Type <return> to continue, or q <return> to quit--- Thread 3 (Thread 0x7f79bdffb700 (LWP 17373)): #0 process_runnable_tasks () at src/task.c:374 #1 0x000055f59f33534f in run_poll_loop () at src/haproxy.c:2667 #2 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #3 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x7f79bd7fa700 (LWP 17374)): #0 0x00007ffd9ac747aa in clock_gettime () #1 0x00007f79e5e87c6d in clock_gettime () from /lib64/libc.so.6 #2 0x000055f59f2acba6 in now_cpu_time () at include/common/time.h:538 #3 tv_leaving_poll (timeout=<optimized out>, interrupted=0) at include/common/time.h:590 #4 _do_poll (p=<optimized out>, exp=1885071580) at src/ev_epoll.c:169 #5 0x000055f59f335323 in run_poll_loop () at src/haproxy.c:2704 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2736 #7 0x00007f79e7139dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007f79e5e71ead in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7f79e7bacb00 (LWP 17359)): #0 0x000055f59f3bdf40 in task_wakeup (f=512, t=0x55f5dc2ad910) at include/proto/task.h:155 #1 wake_expired_tasks () at src/task.c:209 #2 0x000055f59f335372 in run_poll_loop () at src/haproxy.c:2675 #3 run_thread_poll_loop (data=data@entry=0x55f5a0ed6de0) at src/haproxy.c:2736 #4 0x000055f59f2aa4c5 in main (argc=<optimized out>, argv=0x7ffd9ac41ca8) at src/haproxy.c:3398 Best, -- William
