On 03/16/2016 11:28 AM, Paolo Bonzini wrote: > > > On 16/03/2016 11:10, Fam Zheng wrote: >> These are some ideas originated from analyzing the Christian's crash report >> on >> virtio-blk dataplane torture test: >> >> https://lists.gnu.org/archive/html/qemu-devel/2016-03/msg02093.html >> >> The ideas are mostly inspired/suggested by Paolo. This doesn't fix the bug, >> but >> the first and the last patches seem to make the crash less frequent. Also >> thanks Cornelia Huck for reviewing the draft version posted in that thread. > > I see you have fixed the mutex and started check in patch 4, so perhaps > this fixes the bug. :) Bo or Christian, could you try it out---and if > it works try patches 2 to 4 only? > > Thanks, > > Paolo > Seems to lockup.
Thread 5 (Thread 0x3ff8b2ff910 (LWP 88956)): #0 0x000003ff8c97f13e in syscall () at /lib64/libc.so.6 #1 0x00000000803d52fe in futex_wait (ev=0x80a4a104 <rcu_call_ready_event>, val=4294967295) at /home/cborntra/REPOS/qemu/util/qemu-thread-posix.c:292 #2 0x00000000803d558e in qemu_event_wait (ev=0x80a4a104 <rcu_call_ready_event>) at /home/cborntra/REPOS/qemu/util/qemu-thread-posix.c:399 #3 0x00000000803f2c34 in call_rcu_thread (opaque=0x0) at /home/cborntra/REPOS/qemu/util/rcu.c:250 #4 0x000003ff8ca87c2c in start_thread () at /lib64/libpthread.so.0 #5 0x000003ff8c984c7a in thread_start () at /lib64/libc.so.6 Thread 4 (Thread 0x3ff8aaff910 (LWP 88957)): #0 0x000003ff8c9784d8 in ppoll () at /lib64/libc.so.6 #1 0x00000000802efdca in qemu_poll_ns (fds=0x3ff84002240, nfds=2, timeout=-1) at /home/cborntra/REPOS/qemu/qemu-timer.c:313 #2 0x00000000802f2528 in aio_poll (ctx=0xb9e94050, blocking=true) at /home/cborntra/REPOS/qemu/aio-posix.c:453 #3 0x000000008016392a in iothread_run (opaque=0xb9e93b10) at /home/cborntra/REPOS/qemu/iothread.c:46 #4 0x000003ff8ca87c2c in start_thread () at /lib64/libpthread.so.0 #5 0x000003ff8c984c7a in thread_start () at /lib64/libc.so.6 Thread 3 (Thread 0x3ff888dc910 (LWP 88958)): #0 0x000003ff8ca90cd4 in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x000003ff8ca93e74 in __lll_lock_elision () at /lib64/libpthread.so.0 #2 0x00000000803d49ce in qemu_mutex_lock (mutex=0x8061f260 <qemu_global_mutex>) at /home/cborntra/REPOS/qemu/util/qemu-thread-posix.c:64 #3 0x0000000080060ef4 in qemu_mutex_lock_iothread () at /home/cborntra/REPOS/qemu/cpus.c:1226 #4 0x0000000080156af6 in kvm_arch_handle_exit (cs=0xba23b7f0, run=0x3ff8a200000) at /home/cborntra/REPOS/qemu/target-s390x/kvm.c:2024 #5 0x00000000800815de in kvm_cpu_exec (cpu=0xba23b7f0) at /home/cborntra/REPOS/qemu/kvm-all.c:1921 #6 0x000000008006074c in qemu_kvm_cpu_thread_fn (arg=0xba23b7f0) at /home/cborntra/REPOS/qemu/cpus.c:1050 #7 0x000003ff8ca87c2c in start_thread () at /lib64/libpthread.so.0 #8 0x000003ff8c984c7a in thread_start () at /lib64/libc.so.6 Thread 2 (Thread 0x3ff67fff910 (LWP 88959)): #0 0x000003ff8ca90d04 in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x000003ff8ca93e74 in __lll_lock_elision () at /lib64/libpthread.so.0 #2 0x00000000803d49ce in qemu_mutex_lock (mutex=0x8061f260 <qemu_global_mutex>) at /home/cborntra/REPOS/qemu/util/qemu-thread-posix.c:64 #3 0x0000000080060ef4 in qemu_mutex_lock_iothread () at /home/cborntra/REPOS/qemu/cpus.c:1226 #4 0x0000000080156af6 in kvm_arch_handle_exit (cs=0xb9f2e970, run=0x3ff88080000) at /home/cborntra/REPOS/qemu/target-s390x/kvm.c:2024 #5 0x00000000800815de in kvm_cpu_exec (cpu=0xb9f2e970) at /home/cborntra/REPOS/qemu/kvm-all.c:1921 #6 0x000000008006074c in qemu_kvm_cpu_thread_fn (arg=0xb9f2e970) at /home/cborntra/REPOS/qemu/cpus.c:1050 #7 0x000003ff8ca87c2c in start_thread () at /lib64/libpthread.so.0 #8 0x000003ff8c984c7a in thread_start () at /lib64/libc.so.6 Thread 1 (Thread 0x3ff8e55bb90 (LWP 88953)): #0 0x000003ff8ca90cd4 in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x000003ff8ca93e74 in __lll_lock_elision () at /lib64/libpthread.so.0 #2 0x00000000803d49ce in qemu_mutex_lock (mutex=0xba232df8) at /home/cborntra/REPOS/qemu/util/qemu-thread-posix.c:64 #3 0x00000000800b713e in virtio_blk_data_plane_start (s=0xba232d80) at /home/cborntra/REPOS/qemu/hw/block/dataplane/virtio-blk.c:224 #4 0x00000000800b4ea0 in virtio_blk_handle_output (vdev=0xb9eee7e8, vq=0xba305270) at /home/cborntra/REPOS/qemu/hw/block/virtio-blk.c:590 #5 0x00000000800ef3dc in virtio_queue_notify_vq (vq=0xba305270) at /home/cborntra/REPOS/qemu/hw/virtio/virtio.c:1095 #6 0x00000000800f1c9c in virtio_queue_host_notifier_read (n=0xba3052c8) at /home/cborntra/REPOS/qemu/hw/virtio/virtio.c:1785 #7 0x00000000800f1e14 in virtio_queue_set_host_notifier_fd_handler (vq=0xba305270, assign=false, set_handler=false) at /home/cborntra/REPOS/qemu/hw/virtio/virtio.c:1817 #8 0x0000000080109c50 in virtio_ccw_set_guest2host_notifier (dev=0xb9eed6a0, n=0, assign=false, set_handler=false) at /home/cborntra/REPOS/qemu/hw/s390x/virtio-ccw.c:97 #9 0x0000000080109ef2 in virtio_ccw_stop_ioeventfd (dev=0xb9eed6a0) at /home/cborntra/REPOS/qemu/hw/s390x/virtio-ccw.c:154 #10 0x000000008010d5aa in virtio_ccw_set_host_notifier (d=0xb9eed6a0, n=0, assign=true) at /home/cborntra/REPOS/qemu/hw/s390x/virtio-ccw.c:1211 #11 0x00000000800b722c in virtio_blk_data_plane_start (s=0xba232d80) at /home/cborntra/REPOS/qemu/hw/block/dataplane/virtio-blk.c:242 #12 0x00000000800b4ea0 in virtio_blk_handle_output (vdev=0xb9eee7e8, vq=0xba305270) at /home/cborntra/REPOS/qemu/hw/block/virtio-blk.c:590 #13 0x00000000800ef3dc in virtio_queue_notify_vq (vq=0xba305270) at /home/cborntra/REPOS/qemu/hw/virtio/virtio.c:1095 #14 0x00000000800f1c9c in virtio_queue_host_notifier_read (n=0xba3052c8) at /home/cborntra/REPOS/qemu/hw/virtio/virtio.c:1785 #15 0x00000000802f1cd4 in aio_dispatch (ctx=0xb9e81c70) at /home/cborntra/REPOS/qemu/aio-posix.c:327 #16 0x00000000802df31c in aio_ctx_dispatch (source=0xb9e81c70, callback=0x0, user_data=0x0) at /home/cborntra/REPOS/qemu/async.c:232 ---Type <return> to continue, or q <return> to quit--- #17 0x000003ff8d2d1c0a in g_main_context_dispatch () at /lib64/libglib-2.0.so.0 #18 0x00000000802ee9a6 in glib_pollfds_poll () at /home/cborntra/REPOS/qemu/main-loop.c:212 #19 0x00000000802eeae2 in os_host_main_loop_wait (timeout=979000000) at /home/cborntra/REPOS/qemu/main-loop.c:257 #20 0x00000000802eebee in main_loop_wait (nonblocking=0) at /home/cborntra/REPOS/qemu/main-loop.c:505 #21 0x000000008017b4a4 in main_loop () at /home/cborntra/REPOS/qemu/vl.c:1933 #22 0x0000000080183992 in main (argc=72, argv=0x3ffebc7e908, envp=0x3ffebc7eb50) at /home/cborntra/REPOS/qemu/vl.c:4656