Hi,
I wasn't sure who to email but I believe this is somehow related to
the perf counters implementation. I kept getting lockups (deadlocks)
on a 64-bit uniprocessor system:
Linux ubuntu 3.19.0-43-generic #49~14.04.1-Ubuntu SMP Thu Dec 31
15:44:49 UTC 2015 x86_64 x86_64 x86_64 GNU/Linux
This issue seems to be reproducible on all 3.2+ < 4 kernels (possibly
earlier). On my test system, it can take from a few seconds to an hour
for a deadlock to occur.
There's some race condition (?) in x86_pmu_stop() that calls
x86_perf_event_update() that causes it to run with a user-space
pointer. The backtrace is attached. In frame #19, the event pointer is
valid and points to 0xffff88003b4ae000. However, in frame #18,
x86_perf_event_update() is called with the user-space pointer
0x40000002 in %rdi. Same applies to the vfs_write call. If this
address 0x40000002 is mmaped in user space, it might be possible to
perform arbitrary code execution via a crafted file struct, for
example.
The following disassembly (last 2 lines) shows that the call at
0xffffffff8102ba8b is executed before the value in %r12
(0xffff88003b4ae000) is copied into %rdi. So 86_perf_event_update()
runs with the old %rdi value (0x40000002). I'm not sure where this
value 0x40000002 is coming from though. The only explanation I could
find is it could be related to cpuid/rdmsr and the vmware hypervisor
since this was only reproducible on VMware hypervisors. I've also
tried qemu and virtualbox but wasn't able to reproduce it there.
(gdb) disassemble x86_pmu_stop
Dump of assembler code for function x86_pmu_stop:
0xffffffff8102ba30 <+0>: push %rbp
0xffffffff8102ba31 <+1>: mov %rsp,%rbp
0xffffffff8102ba34 <+4>: push %r13
0xffffffff8102ba36 <+6>: mov %esi,%r13d
0xffffffff8102ba39 <+9>: push %r12
0xffffffff8102ba3b <+11>: mov %rdi,%r12
0xffffffff8102ba3e <+14>: push %rbx
0xffffffff8102ba3f <+15>: mov $0xbb20,%rbx
0xffffffff8102ba46 <+22>: sub $0x8,%rsp
0xffffffff8102ba4a <+26>: add %gs:0x7efde6f6(%rip),%rbx
# 0xa148 <this_cpu_off>
0xffffffff8102ba52 <+34>: movslq 0x154(%rdi),%rax
0xffffffff8102ba59 <+41>: btr %rax,0x200(%rbx)
0xffffffff8102ba61 <+49>: sbb %eax,%eax
0xffffffff8102ba63 <+51>: test %eax,%eax
0xffffffff8102ba65 <+53>: jne 0xffffffff8102baa8 <x86_pmu_stop+120>
0xffffffff8102ba67 <+55>: and $0x4,%r13d
0xffffffff8102ba6b <+59>: je 0xffffffff8102ba78 <x86_pmu_stop+72>
0xffffffff8102ba6d <+61>: testb $0x2,0x198(%r12)
0xffffffff8102ba76 <+70>: je 0xffffffff8102ba88 <x86_pmu_stop+88>
0xffffffff8102ba78 <+72>: add $0x8,%rsp
0xffffffff8102ba7c <+76>: pop %rbx
0xffffffff8102ba7d <+77>: pop %r12
0xffffffff8102ba7f <+79>: pop %r13
0xffffffff8102ba81 <+81>: pop %rbp
0xffffffff8102ba82 <+82>: retq
0xffffffff8102ba83 <+83>: nopl 0x0(%rax,%rax,1)
0xffffffff8102ba88 <+88>: mov %r12,%rdi
0xffffffff8102ba8b <+91>: callq 0xffffffff8102b990
<x86_perf_event_update>
Please let me know if you need any other information.
--
--
Regards,
Vitaly
arch_spin_lock (lock=<optimized out>) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/arch/x86/include/asm/spinlock.h:114
114 inc.head = READ_ONCE(lock->tickets.head);
(gdb) bt
#0 arch_spin_lock (lock=<optimized out>) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/arch/x86/include/asm/spinlock.h:114
#1 do_raw_spin_lock (lock=<optimized out>) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/include/linux/spinlock.h:158
#2 __raw_spin_lock_irq (lock=<optimized out>) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/include/linux/spinlock_api_smp.h:129
#3 _raw_spin_lock_irq (lock=0xffff8800324db808) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/locking/spinlock.c:167
#4 0xffffffff811688a8 in perf_remove_from_context (event=0xffff88003b4ae000,
detach_group=true)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/events/core.c:1578
#5 0xffffffff81168a58 in put_event (event=0xffff88003b4ae000) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/events/core.c:3465
#6 0xffffffff81168ad0 in perf_release (inode=<optimized out>, file=<optimized
out>)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/events/core.c:3480
#7 0xffffffff811ee117 in __fput (file=0xffff88003b610c00) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/fs/file_table.c:208
#8 0xffffffff811ee29e in ____fput (work=<optimized out>) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/fs/file_table.c:244
#9 0xffffffff81091eac in task_work_run () at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/task_work.c:115
#10 0xffffffff810773f0 in exit_task_work (task=<optimized out>) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/include/linux/task_work.h:21
#11 do_exit (code=-131940442180816) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/exit.c:745
#12 0xffffffff810187f8 in oops_end (flags=70, regs=0xffff88003257b788, signr=11)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/arch/x86/kernel/dumpstack.c:249
#13 0xffffffff81018deb in die (str=0xffffffff81a7dad7 "general protection
fault", regs=0xffff88003257b788, err=0)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/arch/x86/kernel/dumpstack.c:312
#14 0xffffffff81015916 in do_general_protection (regs=0xffff88003257b788,
error_code=0)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/arch/x86/kernel/traps.c:398
#15 <signal handler called>
#16 native_read_pmc (counter=1073741826) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/arch/x86/include/asm/msr.h:126
#17 0xffffffff8102b9d3 in paravirt_read_pmc (counter=<optimized out>) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/arch/x86/include/asm/paravirt.h:206
#18 x86_perf_event_update (event=0x40000002) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/arch/x86/kernel/cpu/perf_event.c:81
#19 0xffffffff8102ba90 in x86_pmu_stop (event=0xffff88003b4ae000, flags=4)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/arch/x86/kernel/cpu/perf_event.c:1171
#20 0xffffffff8102bb5a in x86_pmu_del (event=0xffff88003b4ae000,
flags=<optimized out>)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/arch/x86/kernel/cpu/perf_event.c:1200
#21 0xffffffff81169416 in event_sched_out (event=0xffff88003b4ae000,
ctx=0xffff8800324db800, cpuctx=<optimized out>, cpuctx=<optimized out>)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/events/core.c:1469
#22 0xffffffff81169598 in group_sched_out (group_event=<optimized out>,
cpuctx=<optimized out>, ctx=0xffff8800324db800)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/events/core.c:1494
#23 0xffffffff81169984 in ctx_sched_out (ctx=0xffff8800324db800,
cpuctx=0xffff88003d617ca0, event_type=EVENT_ALL)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/events/core.c:2253
#24 0xffffffff8116b7ab in perf_event_context_sched_out (next=<optimized out>,
ctxn=<optimized out>, task=<optimized out>)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/events/core.c:2434
#25 __perf_event_task_sched_out (task=0xffff880038cfebf0,
next=0xffff88003c986bf0) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/events/core.c:2460
#26 0xffffffff8109c903 in perf_event_task_sched_out (prev=0xffff880038cfebf0,
next=0xffff88003c986bf0)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/include/linux/perf_event.h:712
#27 0xffffffff817b2d59 in prepare_task_switch (next=<optimized out>,
prev=<optimized out>, rq=<optimized out>)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/sched/core.c:2231
#28 context_switch (next=<optimized out>, prev=<optimized out>, rq=<optimized
out>) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/sched/core.c:2355
#29 __schedule () at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/sched/core.c:2850
#30 0xffffffff817b3727 in __cond_resched () at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/sched/core.c:4262
#31 _cond_resched () at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/sched/core.c:4269
#32 0xffffffff817b6219 in console_conditional_schedule () at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/kernel/printk/printk.c:2303
#33 0xffffffff8141a0bf in fbcon_redraw (vc=<optimized out>, line=<optimized
out>, count=<optimized out>, offset=<optimized out>, p=<optimized out>)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/drivers/video/console/fbcon.c:1755
#34 0xffffffff8141b63b in fbcon_scroll (vc=0xffff88003f820c00, t=0,
b=<optimized out>, dir=<optimized out>, count=1)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/drivers/video/console/fbcon.c:1898
#35 0xffffffff814a7dbc in scrup (vc=0xffff88003f820c00, t=0, b=37, nr=1) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/drivers/tty/vt/vt.c:324
#36 0xffffffff814a7e50 in lf (vc=0xffff88003f820c00) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/drivers/tty/vt/vt.c:1119
#37 0xffffffff814ad6d0 in do_con_write (tty=<optimized out>, buf=<optimized
out>, count=<optimized out>)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/drivers/tty/vt/vt.c:2377
#38 0xffffffff814adb65 in do_con_write (count=<optimized out>, buf=<optimized
out>, tty=<optimized out>)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/drivers/tty/vt/vt.c:2776
#39 con_write (tty=0xffff88003b5e2800, buf=<optimized out>, count=<optimized
out>) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/drivers/tty/vt/vt.c:2772
#40 0xffffffff81495b9d in process_output_block (nr=<optimized out>,
buf=<optimized out>, tty=<optimized out>)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/drivers/tty/n_tty.c:632
#41 n_tty_write (tty=0xffff88003b5e2800, file=<optimized out>,
buf=0xffff88003b5e6400
"\\2\\0\\0\\0\\0\\10\\0\\0\\ffffffd4\\8\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\3\\0\\0\\0\\0\\0\\0\\0\\2\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0"...,
nr=233) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/drivers/tty/n_tty.c:2397
#42 0xffffffff81491ce8 in do_tty_write (count=<optimized out>, buf=<optimized
out>, file=<optimized out>, tty=<optimized out>, write=<optimized out>)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/drivers/tty/tty_io.c:1161
#43 tty_write (file=0xffff880036163400, buf=0x7f695bb08000 <error: Cannot
access memory at address 0x7f695bb08000>, count=<optimized out>,
ppos=<optimized out>)
at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/drivers/tty/tty_io.c:1247
#44 0xffffffff811ec5b7 in vfs_write (file=0x40000002, buf=0x7f695bb08000
<error: Cannot access memory at address 0x7f695bb08000>, count=<optimized out>,
pos=0xffff88003257bf50) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/fs/read_write.c:546
#45 0xffffffff811ed1c6 in SYSC_write (count=<optimized out>, buf=<optimized
out>, fd=<optimized out>)
at /build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/fs/read_write.c:597
#46 SyS_write (fd=<optimized out>, buf=140090486587392, count=233) at
/build/linux-lts-vivid-xAeXSD/linux-lts-vivid-3.19.0/fs/read_write.c:589
#47 <signal handler called>
#48 0x00007f695b60f870 in ?? ()
---Type <return> to continue, or q <return> to quit---
#49 0x00007f109ed36000 in ?? ()
#50 0x00007f109ed37000 in ?? ()
#51 0xffff88003257d380 in ?? ()
#52 0xffff88003257db00 in ?? ()
#53 0xffff88003257d6a1 in ?? ()
#54 0xffff88003257d3a0 in ?? ()
#55 0xffff88003257db20 in ?? ()
#56 0x0000000000000000 in ?? ()
