On Fri, Jan 02, 2015 at 02:35:26PM +0200, Roman Gershman wrote:
> Unfortunately I do not have 50L sample for this but the stacktrace should
> give the clear picture on how this happens.
>
> I solved the problem by moving my shutdown flow (i.e. calls like
> bufferevent_disable) into event base main thread which apparently solves
> the deadlock bug.
Okay, finally, I have some time for validating this, once and for all, and I
confirmed that for 2.0 there is deadlock and it is pretty simple to reproduce,
while for 2.1 there is no deadlock.
It took me some time, since I used bufferevent_pair firstly, but then I
realized that it uses BEV_OPT_DEFER_CALLBACKS.
2.0:
===
$ le20=libevent-2.0.20/.libs
$ gcc -Wl,-rpath=$le20 -L$le20 -g3 -pthread -levent_pthreads -levent test.c -o
test
$ ./test # hanged &
$ gdb -q -p $(pgrep test) -batch -ex 'thread apply all bt' -ex detach
[New LWP 25717]
[New LWP 25716]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
0x00007f8f1604a4db in pthread_join (threadid=140252503488256,
thread_return=0x0) at pthread_join.c:92
Thread 3 (Thread 0x7f8f15264700 (LWP 25716)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at pthread_cond_wait.S:185
#1 0x00007f8f164b3082 in evthread_posix_cond_wait (_cond=0x1cce560,
_lock=0x1cce530, tv=0x0) at evthread_pthread.c:156
#2 0x00007f8f16270570 in event_del_internal (ev=0x1cceb40) at event.c:2211
#3 0x00007f8f162703ef in event_del (ev=0x1cceb40) at event.c:2179
#4 0x00007f8f1627d5dc in be_socket_disable (bufev=0x1cceb30, event=2) at
bufferevent_sock.c:564
#5 0x00007f8f1627b8d1 in bufferevent_disable (bufev=0x1cceb30, event=2) at
bufferevent.c:505
#6 0x000000000040103b in endiscb (arg=0x0) at test.c:53
#7 0x00007f8f160490a4 in start_thread (arg=0x7f8f15264700) at
pthread_create.c:309
#8 0x00007f8f15d7dccd in clone () at clone.S:111
Thread 2 (Thread 0x7f8f14a63700 (LWP 25717)):
#0 __lll_lock_wait () at lowlevellock.S:135
#1 0x00007f8f1604b4d4 in _L_lock_952 () from
/lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007f8f1604b336 in __GI___pthread_mutex_lock (mutex=0x1ccee70) at
pthread_mutex_lock.c:114
#3 0x00007f8f164b2e80 in evthread_posix_lock (mode=0, _lock=0x1ccee70) at
evthread_pthread.c:73
#4 0x00007f8f1627bc1c in _bufferevent_incref_and_lock (bufev=0x1cceb30) at
bufferevent.c:582
#5 0x00007f8f1627c9a3 in bufferevent_readcb (fd=8, event=2, arg=0x1cceb30) at
bufferevent_sock.c:133
#6 0x00007f8f1626d9a0 in event_persist_closure (base=0x1cce0d0, ev=0x1cceb40)
at event.c:1297
#7 0x00007f8f1626daaf in event_process_active_single_queue (base=0x1cce0d0,
activeq=0x1cce510) at event.c:1341
#8 0x00007f8f1626dd7c in event_process_active (base=0x1cce0d0) at event.c:1416
#9 0x00007f8f1626e3e7 in event_base_loop (base=0x1cce0d0, flags=4) at
event.c:1617
#10 0x0000000000401164 in loopcb (arg=0x0) at test.c:76
#11 0x00007f8f160490a4 in start_thread (arg=0x7f8f14a63700) at
pthread_create.c:309
#12 0x00007f8f15d7dccd in clone () at clone.S:111
Thread 1 (Thread 0x7f8f168a1700 (LWP 25714)):
#0 0x00007f8f1604a4db in pthread_join (threadid=140252503488256,
thread_return=0x0) at pthread_join.c:92
#1 0x00000000004013a9 in main (argc=1, argv=0x7fff509b7d98) at test.c:110
2.1:
===
$ gcc -g3 -pthread test.c libevent-github/.cmake-debug/lib/libevent.a -o test
$ ./test
stat> read: 0, readed: 0, write: 1 (0), disabled: 2
stat> read: 0, readed: 0, write: 100001 (0), disabled: 94612
stat> read: 0, readed: 0, write: 200001 (0), disabled: 188108
stat> read: 0, readed: 0, write: 300001 (0), disabled: 304106
stat> read: 0, readed: 0, write: 400001 (0), disabled: 383880
stat> read: 0, readed: 0, write: 500001 (0), disabled: 493574
stat> read: 0, readed: 0, write: 600001 (0), disabled: 568470
stat> read: 1709, readed: 7000000, write: 700000 (0), disabled: 3541154
(Also I've fixed some minor bugs during digging this, IOW they don't affect
behaviour of this particular deadlock).
Cheers,
Azat.
#define _GNU_SOURCE
#include <pthread.h>
#include <assert.h>
#include <event2/bufferevent.h>
#include <event2/buffer.h>
#include <event2/thread.h>
#include <event2/event.h>
#define ITERATIONS 700000
#define ITERATIONS_SIZE 10
struct event_base *base;
struct bufferevent *pair[2];
struct evstat
{
size_t read;
size_t readed;
size_t write;
size_t disable;
size_t free;
} stat;
void prstat()
{
fprintf(stderr, "stat> read: %zu, readed: %zu, write: %zu (%zu), disabled: %zu\n",
stat.read, stat.readed, stat.write, evbuffer_get_length(bufferevent_get_input(pair[1])), stat.disable);
}
void readcb(struct bufferevent *bev, void *arg)
{
struct evbuffer *evbuf;
assert(evbuf = evbuffer_new());
bufferevent_read_buffer(bev, evbuf);
stat.readed += evbuffer_get_length(evbuf);
evbuffer_free(evbuf);
++stat.read;
if (stat.readed == (ITERATIONS * ITERATIONS_SIZE)) {
event_base_loopexit(base, 0);
}
}
void *endiscb(void *arg)
{
size_t i;
for (i = 0; ; ++i) {
if (stat.free) {
break;
}
if (!(i % 100)) {
assert(!bufferevent_disable(pair[1], EV_READ));
assert(!bufferevent_enable(pair[1], EV_READ));
++stat.disable;
}
}
}
void *writercb(void *arg)
{
char buffer[ITERATIONS_SIZE] = { 0 };
size_t i;
for (i = 0; i < ITERATIONS; ++i) {
bufferevent_write(pair[0], buffer, sizeof(buffer));
++stat.write;
if (!(i % 100000)) {
prstat();
}
}
}
void *loopcb(void *arg)
{
event_base_loop(base, EVLOOP_NO_EXIT_ON_EMPTY);
}
void firstcb(evutil_socket_t fd , short what, void *arg)
{
sleep(5);
}
int main(int argc, char **argv)
{
pthread_t writer;
pthread_t endis;
pthread_t loop;
evutil_socket_t pipe[2];
evthread_use_pthreads();
assert(base = event_base_new());
event_base_once(base, -1, EV_TIMEOUT, firstcb, NULL, NULL);
evutil_socketpair(AF_UNIX, SOCK_STREAM, 0, pipe);
evutil_make_socket_nonblocking(pipe[0]);
evutil_make_socket_nonblocking(pipe[1]);
pair[0] = bufferevent_socket_new(base, pipe[0], BEV_OPT_THREADSAFE);
pair[1] = bufferevent_socket_new(base, pipe[1], BEV_OPT_THREADSAFE);
bufferevent_setcb(pair[1], readcb, NULL, NULL, NULL);
assert(!bufferevent_disable(pair[0], EV_READ));
assert(!bufferevent_enable(pair[1], EV_READ));
assert(!pthread_create(&writer, NULL, writercb, NULL));
assert(!pthread_create(&endis, NULL, endiscb, NULL));
assert(!pthread_create(&loop, NULL, loopcb, NULL));
assert(!pthread_join(loop, NULL));
assert(!pthread_join(writer, NULL));
stat.free = 1;
assert(!pthread_join(endis, NULL));
prstat();
bufferevent_free(pair[0]);
bufferevent_free(pair[1]);
event_base_free(base);
#ifdef LIBEVENT_2_1
libevent_global_shutdown();
#endif
return 0;
}
