Hi,
I've encountered deadlock in qemu during some stress testing. The test
is making snapshots, committing them and constantly quering for block
job info.
The version of QEMU is 2.0.0 rc3 (backtrace below says rc2, but it's
manualy patched to rc3), but there seems to be no changes in block layer
in final 2.0 (?).
This is backtrace of qemu process:
(gdb) thread apply all backtrace
Thread 22 (Thread 0x7f6994852700 (LWP 13651)):
#0 0x00007f69982f3d0c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699ab4c4eb in ?? () from /usr/lib64/librados.so.2
#2 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#3 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 21 (Thread 0x7f698ffff700 (LWP 13652)):
#0 0x00007f69982f5ff1 in sem_timedwait () from /lib64/libpthread.so.0
#1 0x00007f699ac3e1b8 in ?? () from /usr/lib64/librados.so.2
#2 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#3 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 20 (Thread 0x7f698f7fe700 (LWP 13653)):
#0 0x00007f69982f3d0c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699ab7b383 in ?? () from /usr/lib64/librados.so.2
#2 0x00007f699abe625d in ?? () from /usr/lib64/librados.so.2
#3 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#4 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 19 (Thread 0x7f698effd700 (LWP 13654)):
#0 0x00007f69982f3d0c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699abe1c88 in ?? () from /usr/lib64/librados.so.2
#2 0x00007f699abe6a6d in ?? () from /usr/lib64/librados.so.2
#3 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#4 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 18 (Thread 0x7f698e7fc700 (LWP 13655)):
#0 0x00007f69982f40de in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699aaeced8 in ?? () from /usr/lib64/librados.so.2
#2 0x00007f699aaede0d in ?? () from /usr/lib64/librados.so.2
#3 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#4 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 17 (Thread 0x7f698dffb700 (LWP 13656)):
#0 0x00007f69982f3d0c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699aaee862 in ?? () from /usr/lib64/librados.so.2
#2 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#3 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 16 (Thread 0x7f698d7fa700 (LWP 13657)):
#0 0x00007f69982f3d0c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699abd288e in ?? () from /usr/lib64/librados.so.2
#2 0x00007f699abddf1d in ?? () from /usr/lib64/librados.so.2
#3 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#4 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 15 (Thread 0x7f698d6f9700 (LWP 13658)):
#0 0x00007f699802007d in poll () from /lib64/libc.so.6
#1 0x00007f699abc56ac in ?? () from /usr/lib64/librados.so.2
#2 0x00007f699abc7460 in ?? () from /usr/lib64/librados.so.2
#3 0x00007f699abd9c2c in ?? () from /usr/lib64/librados.so.2
#4 0x00007f699abde03d in ?? () from /usr/lib64/librados.so.2
#5 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#6 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 14 (Thread 0x7f698d5f8700 (LWP 13659)):
#0 0x00007f69982f40de in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699aaeced8 in ?? () from /usr/lib64/librados.so.2
#2 0x00007f699aaede0d in ?? () from /usr/lib64/librados.so.2
#3 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#4 0x00007f6998029dad in clone () from /lib64/libc.so.6
---Type <return> to continue, or q <return> to quit---
Thread 13 (Thread 0x7f698cdf7700 (LWP 13660)):
#0 0x00007f69982f3d0c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699aaee862 in ?? () from /usr/lib64/librados.so.2
#2 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#3 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 12 (Thread 0x7f697ffff700 (LWP 13661)):
#0 0x00007f69982f3d0c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699aaee862 in ?? () from /usr/lib64/librados.so.2
#2 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#3 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7f697f7fe700 (LWP 13662)):
#0 0x00007f69982f40de in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699a5bc666 in ?? () from /usr/lib64/librbd.so.1
#2 0x00007f699a5cf76d in ?? () from /usr/lib64/librbd.so.1
#3 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#4 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7f698c5f6700 (LWP 13663)):
#0 0x00007f69982f3d0c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699abd288e in ?? () from /usr/lib64/librados.so.2
#2 0x00007f699abddf1d in ?? () from /usr/lib64/librados.so.2
#3 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#4 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7f698c4f5700 (LWP 13664)):
#0 0x00007f699802007d in poll () from /lib64/libc.so.6
#1 0x00007f699abc56ac in ?? () from /usr/lib64/librados.so.2
#2 0x00007f699abc7460 in ?? () from /usr/lib64/librados.so.2
#3 0x00007f699abd9c2c in ?? () from /usr/lib64/librados.so.2
#4 0x00007f699abde03d in ?? () from /usr/lib64/librados.so.2
#5 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#6 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7f697effd700 (LWP 13665)):
#0 0x00007f69982f67a4 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f69982f219c in _L_lock_518 () from /lib64/libpthread.so.0
#2 0x00007f69982f1feb in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007f699c332df9 in qemu_mutex_lock
(mutex=mutex@entry=0x7f699cb87660 <qemu_global_mutex>) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/util/qemu-thread-posix.c:76
#4 0x00007f699c238a10 in qemu_mutex_lock_iothread () at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/cpus.c:1043
#5 0x00007f699c291f3d in kvm_cpu_exec (cpu=cpu@entry=0x7f699f03d090) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/kvm-all.c:1683
#6 0x00007f699c23771c in qemu_kvm_cpu_thread_fn (arg=0x7f699f03d090) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/cpus.c:873
#7 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#8 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7f697e7fc700 (LWP 13666)):
#0 0x00007f69982f67a4 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f69982f219c in _L_lock_518 () from /lib64/libpthread.so.0
#2 0x00007f69982f1feb in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007f699c332df9 in qemu_mutex_lock
(mutex=mutex@entry=0x7f699cb87660 <qemu_global_mutex>) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/util/qemu-thread-posix.c:76
#4 0x00007f699c238a10 in qemu_mutex_lock_iothread () at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/cpus.c:1043
#5 0x00007f699c291f3d in kvm_cpu_exec (cpu=cpu@entry=0x7f699f0774a0) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/kvm-all.c:1683
#6 0x00007f699c23771c in qemu_kvm_cpu_thread_fn (arg=0x7f699f0774a0) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/cpus.c:873
#7 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#8 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7f697cfff700 (LWP 13668)):
#0 0x00007f69982f3d0c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699c333019 in qemu_cond_wait
(cond=cond@entry=0x7f699f15fba0, mutex=mutex@entry=0x7f699f15fbd0) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/util/qemu-thread-posix.c:135
#2 0x00007f699c22770b in vnc_worker_thread_loop
(queue=queue@entry=0x7f699f15fba0) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/ui/vnc-jobs.c:222
#3 0x00007f699c227ae0 in vnc_worker_thread (arg=0x7f699f15fba0) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/ui/vnc-jobs.c:323
#4 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#5 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7f6995458700 (LWP 23551)):
#0 0x00007f69982f3d0c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699abd288e in ?? () from /usr/lib64/librados.so.2
#2 0x00007f699abddf1d in ?? () from /usr/lib64/librados.so.2
#3 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#4 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f697dffb700 (LWP 23552)):
#0 0x00007f699802007d in poll () from /lib64/libc.so.6
#1 0x00007f699abc56ac in ?? () from /usr/lib64/librados.so.2
#2 0x00007f699abc7460 in ?? () from /usr/lib64/librados.so.2
#3 0x00007f699abd9c2c in ?? () from /usr/lib64/librados.so.2
#4 0x00007f699abde03d in ?? () from /usr/lib64/librados.so.2
#5 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#6 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f6995056700 (LWP 23553)):
#0 0x00007f69982f3d0c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f699abd288e in ?? () from /usr/lib64/librados.so.2
#2 0x00007f699abddf1d in ?? () from /usr/lib64/librados.so.2
#3 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#4 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f697ddff700 (LWP 23554)):
#0 0x00007f699802007d in poll () from /lib64/libc.so.6
#1 0x00007f699abc56ac in ?? () from /usr/lib64/librados.so.2
#2 0x00007f699abc7460 in ?? () from /usr/lib64/librados.so.2
#3 0x00007f699abd9c2c in ?? () from /usr/lib64/librados.so.2
#4 0x00007f699abde03d in ?? () from /usr/lib64/librados.so.2
#5 0x00007f69982eff3a in start_thread () from /lib64/libpthread.so.0
#6 0x00007f6998029dad in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f699bfcd900 (LWP 13647)):
#0 0x00007f6998020286 in ppoll () from /lib64/libc.so.6
#1 0x00007f699c1f3d9b in ppoll (__ss=0x0, __timeout=0x0,
__nfds=<optimized out>, __fds=<optimized out>) at
/usr/include/bits/poll2.h:77
#2 qemu_poll_ns (fds=<optimized out>, nfds=<optimized out>,
timeout=<optimized out>) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/qemu-timer.c:311
#3 0x00007f699c0877e0 in aio_poll (ctx=0x7f699e4c9c00,
blocking=blocking@entry=true) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/aio-posix.c:221
#4 0x00007f699c095c0a in bdrv_drain_all () at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/block.c:1805
#5 0x00007f699c09c87e in bdrv_close (bs=bs@entry=0x7f699f0bc520) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/block.c:1695
#6 0x00007f699c09c5fa in bdrv_delete (bs=0x7f699f0bc520) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/block.c:1978
#7 bdrv_unref (bs=0x7f699f0bc520) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/block.c:5198
#8 0x00007f699c09c812 in bdrv_drop_intermediate
(active=active@entry=0x7f699ebfd330, top=top@entry=0x7f699f0bc520,
base=base@entry=0x7f699eec43d0) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/block.c:2567
#9 0x00007f699c0a1963 in commit_run (opaque=0x7f699f17dcc0) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/block/commit.c:144
#10 0x00007f699c0e0dca in coroutine_trampoline (i0=<optimized out>,
i1=<optimized out>) at
/var/tmp/portage/app-emulation/qemu-2.0.0_rc2/work/qemu-2.0.0-rc2/coroutine-ucontext.c:118
#11 0x00007f6997f859f0 in ?? () from /lib64/libc.so.6
#12 0x00007fffdbe06750 in ?? ()
#13 0x0000000000000000 in ?? ()
--
mg
