Hi,
I have more info on this. It seems that this problem occurs when I bring a
haproxy server up and there's already an existing haproxy server in the
peer group
--> Server A is running and handles requests
--> Server B comes up and immediately dies.
I believe if both servers are already up and the request using the stick
table goes to A, B dies at that point as well. (Need to check this case
further).
One thing to note is that I am using two stick tables used by about 10 back
ends. Excerpt:
peers mypeers
peer www2-new 10.0.3.174:1024
peer www1-new 10.0.2.85:1024
backend simulate
option httpchk OPTIONS /simulate/api/status
stick-table type string len 40 size 5M expire 30m peers mypeers
stick store-response set-cookie(SIMULATE_STICKY_SESSION) table simulate
stick on cookie(SIMULATE_STICKY_SESSION) table simulate
stick on url_param(SIMULATE_STICKY_SESSION,;) table simulate
server app1 10.0.2.11:8080 cookie app1 check inter 10000
server app2 10.0.3.11:8080 cookie app2 check inter 10000
backend simulate-qa
option httpchk OPTIONS /simulate/api/status
stick-table type string len 40 size 5M expire 30m peers mypeers
stick store-response set-cookie(SIMULATE_STICKY_SESSION) table
simulate-qa
stick on cookie(SIMULATE_STICKY_SESSION) table simulate-qa
stick on url_param(SIMULATE_STICKY_SESSION,;) table simulate-qa
server qa1 10.0.2.125:8080 cookie qa1 check inter 10000
server qa2 10.0.3.125:8080 cookie qa2 check inter 10000
Here's the gdb output, following instructions from Lukas.
-------------------
gdb ./haproxy /tmp/core-haproxy-6-117-126-25675-1365638027
Reading symbols from /home/wglass/haproxy-1.5-dev18/haproxy...done.
[New LWP 25675]
warning: Can't read pathname for load map: Input/output error.
Core was generated by `./haproxy -f /etc/haproxy/haproxy.cfg -D -p
/var/run/haproxy.pid'.
Program terminated with signal 6, Aborted.
#0 0x00007f1c2fd5c425 in raise () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) bt
#0 0x00007f1c2fd5c425 in raise () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f1c2fd5fb8b in abort () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00007f1c2fd9a39e in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#3 0x00007f1c2fe30807 in __fortify_fail () from
/lib/x86_64-linux-gnu/libc.so.6
#4 0x00007f1c2fe307d0 in __stack_chk_fail () from
/lib/x86_64-linux-gnu/libc.so.6
#5 0x000000000043f902 in peer_io_handler (si=0xeec028) at src/peers.c:1041
#6 0x00000000004843f4 in process_session (t=0xeec4b0) at src/session.c:2385
#7 0x0000000000412e5c in process_runnable_tasks (next=0x7fffc6227edc) at
src/task.c:238
#8 0x00000000004081d0 in run_poll_loop () at src/haproxy.c:1210
#9 0x0000000000408d16 in main (argc=6, argv=0x7fffc6228138) at
src/haproxy.c:1541
(gdb) backtrace full
#0 0x00007f1c2fd5c425 in raise () from /lib/x86_64-linux-gnu/libc.so.6
No symbol table info available.
#1 0x00007f1c2fd5fb8b in abort () from /lib/x86_64-linux-gnu/libc.so.6
No symbol table info available.
#2 0x00007f1c2fd9a39e in ?? () from /lib/x86_64-linux-gnu/libc.so.6
No symbol table info available.
#3 0x00007f1c2fe30807 in __fortify_fail () from
/lib/x86_64-linux-gnu/libc.so.6
No symbol table info available.
#4 0x00007f1c2fe307d0 in __stack_chk_fail () from
/lib/x86_64-linux-gnu/libc.so.6
No symbol table info available.
#5 0x000000000043f902 in peer_io_handler (si=0xeec028) at src/peers.c:1041
t = 0xeec4b0
s = 0xeebda0
curpeers = 0xd19180
reql = 0
repl = 5
#6 0x00000000004843f4 in process_session (t=0xeec4b0) at src/session.c:2385
srv = 0x0
s = 0xeebda0
rqf_last = 8421376
rpf_last = 32770
rq_prod_last = 7
rq_cons_last = 7
rp_cons_last = 7
rp_prod_last = 7
req_ana_back = 0
__FUNCTION__ = "process_session"
#7 0x0000000000412e5c in process_runnable_tasks (next=0x7fffc6227edc) at
src/task.c:238
t = 0xeec4b0
eb = 0x0
max_processed = 0
expire = -161571788
#8 0x00000000004081d0 in run_poll_loop () at src/haproxy.c:1210
next = -161571788
#9 0x0000000000408d16 in main (argc=6, argv=0x7fffc6228138) at
src/haproxy.c:1541
err = 0
retry = 200
limit = {rlim_cur = 8250, rlim_max = 8250}
errmsg =
"\000\352\322/\034\177\000\000\000\000\000\000\000\000\000\000\360\232/0\034\177\000\000p\204\320\000\000\000\000\000\020\200\"\306\377\177\000\000(\275l\000\000\000\000\000\006\000\000\000\000\000\000\000\065\062\327\060\034\177\000\000p\204\320\000\000\000\000\000\342\001\000\000\000\000\000\000\b",
'\000' <repeats 15 times>"\240, \204", <incomplete sequence \320>
pidfd = 9
On Mon, Apr 8, 2013 at 4:06 AM, Lukas Tribus <[email protected]> wrote:
> Can you recompile with the debug option and without compiler optimizations,
> create a coredump and backtrace it:
>
> # compile (debug + no optimizations)
> make DEBUG=-DDEBUG_FULL CFLAGS="-g -O0" TARGET=[...]
>
> # reise ulimit
> ulimit -c 750000000
>
> # enable coredumping
> echo 2 > /proc/sys/fs/suid_dumpable
>
> # dump core to /tmp/core[...]
> echo "/tmp/core-%e-%s-%u-%g-%p-%t" > /proc/sys/kernel/core_pattern
>
> # install gdb and do backtracing
> apt-get install gdb
> gdb /path/to/haproxy /tmp/core[...]
> bt
> backtrace full
>
>
>
> Then send the output ot gdb (bt and backtrace full).
>
>
>
> Thanks
> Lukas
--
Forio Online Simulations
Will Glass-Husain
415 440-7500x89
[email protected]
www.forio.com
