Re: KASAN: use-after-free Read in cma_cancel_operation, rdma_listen
On Tue, Apr 13, 2021 at 10:19:25PM +0800, Hao Sun wrote: > Jason Gunthorpe 于2021年4月13日周二 下午9:45写道： > > > > On Tue, Apr 13, 2021 at 09:42:43PM +0800, Hao Sun wrote: > > > Jason Gunthorpe 于2021年4月13日周二 下午9:34写道： > > > > > > > > On Tue, Apr 13, 2021 at 11:36:41AM +0800, Hao Sun wrote: > > > > > Hi > > > > > > > > > > When using Healer(https://github.com/SunHao-0/healer/tree/dev) to fuzz > > > > > the Linux kernel, I found two use-after-free bugs which have been > > > > > reported a long time ago by Syzbot. > > > > > Although the corresponding patches have been merged into upstream, > > > > > these two bugs can still be triggered easily. > > > > > The original information about Syzbot report can be found here: > > > > > https://syzkaller.appspot.com/bug?id=8dc0bcd9dd6ec915ba10b3354740eb420884acaa > > > > > https://syzkaller.appspot.com/bug?id=95f89b8fb9fdc42e28ad586e657fea074e4e719b > > > > > > > > Then why hasn't syzbot seen this in a year's time? Seems strange > > > > > > > > > > Seems strange to me too, but the fact is that the reproduction program > > > in attachment can trigger these two bugs quickly. > > > > Do you have this in the C format? > > > > Just tried to use syz-prog2c to convert the repro-prog to C format. > The repro program of rdma_listen was successfully reproduced > (uploaded in attachment), the other one failed. it looks like > syz-prog2c may not be able to do the equivalent conversion. > You can use syz-execprog to execute the reprogram directly, this > method can reproduce both crashes, I have tried it. I tried this program and it reliably deadlocks my kernel :| So there is certainly something here Jason
Re: KASAN: use-after-free Read in cma_cancel_operation, rdma_listen
Jason Gunthorpe 于2021年4月13日周二 下午9:45写道：
>
> On Tue, Apr 13, 2021 at 09:42:43PM +0800, Hao Sun wrote:
> > Jason Gunthorpe 于2021年4月13日周二 下午9:34写道：
> > >
> > > On Tue, Apr 13, 2021 at 11:36:41AM +0800, Hao Sun wrote:
> > > > Hi
> > > >
> > > > When using Healer(https://github.com/SunHao-0/healer/tree/dev) to fuzz
> > > > the Linux kernel, I found two use-after-free bugs which have been
> > > > reported a long time ago by Syzbot.
> > > > Although the corresponding patches have been merged into upstream,
> > > > these two bugs can still be triggered easily.
> > > > The original information about Syzbot report can be found here:
> > > > https://syzkaller.appspot.com/bug?id=8dc0bcd9dd6ec915ba10b3354740eb420884acaa
> > > > https://syzkaller.appspot.com/bug?id=95f89b8fb9fdc42e28ad586e657fea074e4e719b
> > >
> > > Then why hasn't syzbot seen this in a year's time? Seems strange
> > >
> >
> > Seems strange to me too, but the fact is that the reproduction program
> > in attachment can trigger these two bugs quickly.
>
> Do you have this in the C format?
>
Just tried to use syz-prog2c to convert the repro-prog to C format.
The repro program of rdma_listen was successfully reproduced
(uploaded in attachment), the other one failed. it looks like
syz-prog2c may not be able to do the equivalent conversion.
You can use syz-execprog to execute the reprogram directly, this
method can reproduce both crashes, I have tried it.
For reproduction prog of cma_cancel_operation, using syz-execprog
can run this reproduction program directly:
./syz-execprog -threaded -repeat 0 -procs 2 -slowdown 1 -enable tun
-enable netdev -enable resetnet -enable cgroups -enable binfmt-misc
-enable close_fds -enable devlinkpci -enable usb -enable vhci -enable
wifi -enable ieee802154 -enable sysctl repro.prog
// autogenerated by syzkaller (https://github.com/google/syzkaller)
#define _GNU_SOURCE
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
static __thread int skip_segv;
static __thread jmp_buf segv_env;
static void segv_handler(int sig, siginfo_t* info, void* ctx)
{
uintptr_t addr = (uintptr_t)info->si_addr;
const uintptr_t prog_start = 1 << 20;
const uintptr_t prog_end = 100 << 20;
int skip = __atomic_load_n(_segv, __ATOMIC_RELAXED) != 0;
int valid = addr < prog_start || addr > prog_end;
if (skip && valid) {
_longjmp(segv_env, 1);
}
exit(sig);
}
static void install_segv_handler(void)
{
struct sigaction sa;
memset(, 0, sizeof(sa));
sa.sa_handler = SIG_IGN;
syscall(SYS_rt_sigaction, 0x20, , NULL, 8);
syscall(SYS_rt_sigaction, 0x21, , NULL, 8);
memset(, 0, sizeof(sa));
sa.sa_sigaction = segv_handler;
sa.sa_flags = SA_NODEFER | SA_SIGINFO;
sigaction(SIGSEGV, , NULL);
sigaction(SIGBUS, , NULL);
}
#define NONFAILING(...)\
({ \
int ok = 1;\
__atomic_fetch_add(_segv, 1, __ATOMIC_SEQ_CST); \
if (_setjmp(segv_env) == 0) { \
__VA_ARGS__; \
} else \
ok = 0; \
__atomic_fetch_sub(_segv, 1, __ATOMIC_SEQ_CST); \
ok;\
})
static void sleep_ms(uint64_t ms)
{
usleep(ms * 1000);
}
static uint64_t current_time_ms(void)
{
struct timespec ts;
if (clock_gettime(CLOCK_MONOTONIC, ))
exit(1);
return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 100;
}
static void use_temporary_dir(void)
{
char tmpdir_template[] = "./syzkaller.XX";
char* tmpdir = mkdtemp(tmpdir_template);
if (!tmpdir)
exit(1);
if (chmod(tmpdir, 0777))
exit(1);
if (chdir(tmpdir))
exit(1);
}
static bool write_file(const char* file, const char* what, ...)
{
char buf[1024];
va_list args;
va_start(args, what);
vsnprintf(buf, sizeof(buf), what, args);
va_end(args);
buf[sizeof(buf) - 1] = 0;
int len = strlen(buf);
int fd = open(file, O_WRONLY | O_CLOEXEC);
if (fd == -1)
return false;
if (write(fd, buf, len) != len) {
int err = errno;
close(fd);
errno = err;
return false;
}
close(fd);
return true;
}
#define FS_IOC_SETFLAGS _IOW('f', 2, long)
static void remove_dir(const char* dir)
{
int iter = 0;
DIR* dp = 0;
retry:
while (umount2(dir, MNT_DETACH) == 0) {
}
dp = opendir(dir);
if (dp == NULL) {
if (errno == EMFILE) {
exit(1);
}
exit(1);
}
struct dirent* ep = 0;
while ((ep = readdir(dp))) {
if
Re: KASAN: use-after-free Read in cma_cancel_operation, rdma_listen
> On 13 Apr 2021, at 15:44, Jason Gunthorpe wrote: > > On Tue, Apr 13, 2021 at 09:42:43PM +0800, Hao Sun wrote: >> Jason Gunthorpe 于2021年4月13日周二 下午9:34写道： >>> >>> On Tue, Apr 13, 2021 at 11:36:41AM +0800, Hao Sun wrote: Hi When using Healer(https://github.com/SunHao-0/healer/tree/dev) to fuzz the Linux kernel, I found two use-after-free bugs which have been reported a long time ago by Syzbot. Although the corresponding patches have been merged into upstream, these two bugs can still be triggered easily. The original information about Syzbot report can be found here: https://syzkaller.appspot.com/bug?id=8dc0bcd9dd6ec915ba10b3354740eb420884acaa https://syzkaller.appspot.com/bug?id=95f89b8fb9fdc42e28ad586e657fea074e4e719b >>> >>> Then why hasn't syzbot seen this in a year's time? Seems strange >>> >> >> Seems strange to me too, but the fact is that the reproduction program >> in attachment can trigger these two bugs quickly. > > Do you have this in the C format? How can a commit changing arch m68k provoke a bug when running on x86_64? Håkon > > Jason
Re: KASAN: use-after-free Read in cma_cancel_operation, rdma_listen
On Tue, Apr 13, 2021 at 09:42:43PM +0800, Hao Sun wrote: > Jason Gunthorpe 于2021年4月13日周二 下午9:34写道： > > > > On Tue, Apr 13, 2021 at 11:36:41AM +0800, Hao Sun wrote: > > > Hi > > > > > > When using Healer(https://github.com/SunHao-0/healer/tree/dev) to fuzz > > > the Linux kernel, I found two use-after-free bugs which have been > > > reported a long time ago by Syzbot. > > > Although the corresponding patches have been merged into upstream, > > > these two bugs can still be triggered easily. > > > The original information about Syzbot report can be found here: > > > https://syzkaller.appspot.com/bug?id=8dc0bcd9dd6ec915ba10b3354740eb420884acaa > > > https://syzkaller.appspot.com/bug?id=95f89b8fb9fdc42e28ad586e657fea074e4e719b > > > > Then why hasn't syzbot seen this in a year's time? Seems strange > > > > Seems strange to me too, but the fact is that the reproduction program > in attachment can trigger these two bugs quickly. Do you have this in the C format? Jason
Re: KASAN: use-after-free Read in cma_cancel_operation, rdma_listen
Jason Gunthorpe 于2021年4月13日周二 下午9:34写道： > > On Tue, Apr 13, 2021 at 11:36:41AM +0800, Hao Sun wrote: > > Hi > > > > When using Healer(https://github.com/SunHao-0/healer/tree/dev) to fuzz > > the Linux kernel, I found two use-after-free bugs which have been > > reported a long time ago by Syzbot. > > Although the corresponding patches have been merged into upstream, > > these two bugs can still be triggered easily. > > The original information about Syzbot report can be found here: > > https://syzkaller.appspot.com/bug?id=8dc0bcd9dd6ec915ba10b3354740eb420884acaa > > https://syzkaller.appspot.com/bug?id=95f89b8fb9fdc42e28ad586e657fea074e4e719b > > Then why hasn't syzbot seen this in a year's time? Seems strange > Seems strange to me too, but the fact is that the reproduction program in attachment can trigger these two bugs quickly. > > Current report Information: > > commit: 89698becf06d341a700913c3d89ce2a914af69a2 > > What commit is this? The latest commit in the master branch, https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/
Re: KASAN: use-after-free Read in cma_cancel_operation, rdma_listen
On Tue, Apr 13, 2021 at 11:36:41AM +0800, Hao Sun wrote: > Hi > > When using Healer(https://github.com/SunHao-0/healer/tree/dev) to fuzz > the Linux kernel, I found two use-after-free bugs which have been > reported a long time ago by Syzbot. > Although the corresponding patches have been merged into upstream, > these two bugs can still be triggered easily. > The original information about Syzbot report can be found here: > https://syzkaller.appspot.com/bug?id=8dc0bcd9dd6ec915ba10b3354740eb420884acaa > https://syzkaller.appspot.com/bug?id=95f89b8fb9fdc42e28ad586e657fea074e4e719b Then why hasn't syzbot seen this in a year's time? Seems strange > Current report Information: > commit: 89698becf06d341a700913c3d89ce2a914af69a2 What commit is this? Jason
Re: KASAN: use-after-free Read in cma_cancel_operation
On Tue, Apr 3, 2018 at 5:01 AM syzbot wrote: > > Hello, > > syzbot hit the following crash on upstream commit > 86bbbebac1933e6e95e8234c4f7d220c5ddd38bc (Mon Apr 2 18:47:07 2018 +) > Merge branch 'ras-core-for-linus' of > git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip > syzbot dashboard link: > https://syzkaller.appspot.com/bug?extid=6956235342b7317ec564 > > So far this crash happened 4 times on upstream. > C reproducer: https://syzkaller.appspot.com/x/repro.c?id=6621340819783680 > syzkaller reproducer: > https://syzkaller.appspot.com/x/repro.syz?id=6075521074987008 > Raw console output: > https://syzkaller.appspot.com/x/log.txt?id=6447569127342080 > Kernel config: > https://syzkaller.appspot.com/x/.config?id=6801295859785128502 > compiler: gcc (GCC) 7.1.1 20170620 > > IMPORTANT: if you fix the bug, please add the following tag to the commit: > Reported-by: syzbot+6956235342b7317ec...@syzkaller.appspotmail.com > It will help syzbot understand when the bug is fixed. See footer for > details. > If you forward the report, please keep this part and the footer. First crash: 323d, last: 12h23m https://syzkaller.appspot.com/bug?extid=6956235342b7317ec564 > == > BUG: KASAN: use-after-free in __list_del_entry_valid+0x144/0x150 > lib/list_debug.c:54 > Read of size 8 at addr 8801d8dc7d20 by task syzkaller468166/4496 > > CPU: 0 PID: 4496 Comm: syzkaller468166 Not tainted 4.16.0+ #376 > Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS > Google 01/01/2011 > Call Trace: > __dump_stack lib/dump_stack.c:17 [inline] > dump_stack+0x1a7/0x27d lib/dump_stack.c:53 > print_address_description+0x73/0x250 mm/kasan/report.c:256 > kasan_report_error mm/kasan/report.c:354 [inline] > kasan_report+0x23c/0x360 mm/kasan/report.c:412 > __asan_report_load8_noabort+0x14/0x20 mm/kasan/report.c:433 > __list_del_entry_valid+0x144/0x150 lib/list_debug.c:54 > __list_del_entry include/linux/list.h:117 [inline] > list_del include/linux/list.h:125 [inline] > cma_cancel_listens drivers/infiniband/core/cma.c:1569 [inline] > cma_cancel_operation+0x455/0xd60 drivers/infiniband/core/cma.c:1597 > rdma_destroy_id+0xff/0xda0 drivers/infiniband/core/cma.c:1661 > ucma_close+0x100/0x2f0 drivers/infiniband/core/ucma.c:1735 > __fput+0x327/0x7f0 fs/file_table.c:209 > fput+0x15/0x20 fs/file_table.c:243 > task_work_run+0x1ab/0x280 kernel/task_work.c:113 > exit_task_work include/linux/task_work.h:22 [inline] > do_exit+0x1986/0x2700 kernel/exit.c:865 > do_group_exit+0x149/0x400 kernel/exit.c:968 > get_signal+0x74c/0x16e0 kernel/signal.c:2469 > do_signal+0x90/0x1e90 arch/x86/kernel/signal.c:809 > exit_to_usermode_loop+0x258/0x2f0 arch/x86/entry/common.c:162 > prepare_exit_to_usermode arch/x86/entry/common.c:196 [inline] > syscall_return_slowpath arch/x86/entry/common.c:265 [inline] > do_syscall_64+0x6ec/0x940 arch/x86/entry/common.c:292 > entry_SYSCALL_64_after_hwframe+0x42/0xb7 > RIP: 0033:0x446b09 > RSP: 002b:7f2b89150da8 EFLAGS: 0246 ORIG_RAX: 00ca > RAX: fe00 RBX: 006e29fc RCX: 00446b09 > RDX: RSI: RDI: 006e29fc > RBP: 006e29f8 R08: R09: > R10: R11: 0246 R12: 006d635f616d6472 > R13: 2f646e6162696e69 R14: 666e692f7665642f R15: 0005 > > Allocated by task 4493: > save_stack+0x43/0xd0 mm/kasan/kasan.c:447 > set_track mm/kasan/kasan.c:459 [inline] > kasan_kmalloc+0xad/0xe0 mm/kasan/kasan.c:552 > kmem_cache_alloc_trace+0x136/0x740 mm/slab.c:3608 > kmalloc include/linux/slab.h:512 [inline] > kzalloc include/linux/slab.h:701 [inline] > rdma_create_id+0xd0/0x640 drivers/infiniband/core/cma.c:787 > ucma_create_id+0x35f/0x920 drivers/infiniband/core/ucma.c:480 > ucma_write+0x2d6/0x3d0 drivers/infiniband/core/ucma.c:1656 > __vfs_write+0xef/0x970 fs/read_write.c:480 > vfs_write+0x189/0x510 fs/read_write.c:544 > SYSC_write fs/read_write.c:589 [inline] > SyS_write+0xef/0x220 fs/read_write.c:581 > do_syscall_64+0x281/0x940 arch/x86/entry/common.c:287 > entry_SYSCALL_64_after_hwframe+0x42/0xb7 > > Freed by task 4496: > save_stack+0x43/0xd0 mm/kasan/kasan.c:447 > set_track mm/kasan/kasan.c:459 [inline] > __kasan_slab_free+0x11a/0x170 mm/kasan/kasan.c:520 > kasan_slab_free+0xe/0x10 mm/kasan/kasan.c:527 > __cache_free mm/slab.c:3486 [inline] > kfree+0xd9/0x260 mm/slab.c:3801 > rdma_destroy_id+0x821/0xda0 drivers/infiniband/core/cma.c:1691 > ucma_close+0x100/0x2f0 drivers/infiniband/core/ucma.c:1735 > __fput+0x327/0x7f0 fs/file_table.c:209 > fput+0x15/0x20 fs/file_table.c:243 > task_work_run+0x1ab/0x280 kernel/task_work.c:113 > exit_task_work include/linux/task_work.h:22 [inline] > do_exit+0x1986/0x2700 kernel/exit.c:865 > do_group_exit+0x149/0x400 kernel/exit.c:968 > get_signal+0x74c/0x16e0
Re: KASAN: use-after-free Read in cma_cancel_operation
On Sun, May 13, 2018 at 02:34:13PM -0600, Jason Gunthorpe wrote: > On Fri, May 11, 2018 at 02:25:22PM +0900, DaeRyong Jeong wrote: > > We report the crash: KASAN: use-after-free Read in cma_cancel_operation > > > > Note that this bug is previously reported by syzkaller. > > https://syzkaller.appspot.com/bug?id=95f89b8fb9fdc42e28ad586e657fea074e4e719b > > Nonetheless, this bug has not fixed yet, and we hope that this report and > > our > > analysis, which gets help by the RaceFuzzer's feature, will helpful to fix > > the > > crash. > > > > This crash has been found in v4.17-rc1 using RaceFuzzer (a modified > > version of Syzkaller), which we describe more at the end of this > > report. Our analysis shows that the race occurs when invoking two > > syscalls concurrently, write$rdma_cm and write$rdma_cm. > > Well, calling rdma_destroy_id() twice/concurrently is invalid.. The > confusing part of this is how does it happen from ucma.c .. > > Double calls via write look OK to me, the ID is removed from the IDR > at the top so it cannot be invoked twice.. So not sure what > "write$rdma_cm and write$rdma_cm." is supposed to me? It meant two write syscalls. One for ucma_listen, one for ucma_resolve_ip. > > Is your test showing that write() vs close() is the problem? The oops > suggests that.. And the logic around ctx->closing looks tortured > enough that it is probably wrong... My previous dianosis seems wrong... I'm sorry to make you confusing. I'm looking into the code and trying to find out the cause of the crash. > > Jason We have found the one more crash. null-ptr-deref in cma_bind_listen caused by two ucma_listen calls. I think I have the clear idea why null-ptr-deref occured. I will send the report for the null-ptr-deref. Please look at it. We suspect that this crash and null-ptr-deref have the same root cause, incomplete state check. Hopefully, the second report is helpful to find out the root cause of this bug. DaeRyong Jeong
Re: KASAN: use-after-free Read in cma_cancel_operation
On Sun, May 13, 2018 at 02:34:13PM -0600, Jason Gunthorpe wrote: > On Fri, May 11, 2018 at 02:25:22PM +0900, DaeRyong Jeong wrote: > > We report the crash: KASAN: use-after-free Read in cma_cancel_operation > > > > Note that this bug is previously reported by syzkaller. > > https://syzkaller.appspot.com/bug?id=95f89b8fb9fdc42e28ad586e657fea074e4e719b > > Nonetheless, this bug has not fixed yet, and we hope that this report and > > our > > analysis, which gets help by the RaceFuzzer's feature, will helpful to fix > > the > > crash. > > > > This crash has been found in v4.17-rc1 using RaceFuzzer (a modified > > version of Syzkaller), which we describe more at the end of this > > report. Our analysis shows that the race occurs when invoking two > > syscalls concurrently, write$rdma_cm and write$rdma_cm. > > Well, calling rdma_destroy_id() twice/concurrently is invalid.. The > confusing part of this is how does it happen from ucma.c .. > > Double calls via write look OK to me, the ID is removed from the IDR > at the top so it cannot be invoked twice.. So not sure what > "write$rdma_cm and write$rdma_cm." is supposed to me? It meant two write syscalls. One for ucma_listen, one for ucma_resolve_ip. > > Is your test showing that write() vs close() is the problem? The oops > suggests that.. And the logic around ctx->closing looks tortured > enough that it is probably wrong... My previous dianosis seems wrong... I'm sorry to make you confusing. I'm looking into the code and trying to find out the cause of the crash. > > Jason We have found the one more crash. null-ptr-deref in cma_bind_listen caused by two ucma_listen calls. I think I have the clear idea why null-ptr-deref occured. I will send the report for the null-ptr-deref. Please look at it. We suspect that this crash and null-ptr-deref have the same root cause, incomplete state check. Hopefully, the second report is helpful to find out the root cause of this bug. DaeRyong Jeong
Re: KASAN: use-after-free Read in cma_cancel_operation
On Fri, May 11, 2018 at 02:25:22PM +0900, DaeRyong Jeong wrote: > We report the crash: KASAN: use-after-free Read in cma_cancel_operation > > Note that this bug is previously reported by syzkaller. > https://syzkaller.appspot.com/bug?id=95f89b8fb9fdc42e28ad586e657fea074e4e719b > Nonetheless, this bug has not fixed yet, and we hope that this report and our > analysis, which gets help by the RaceFuzzer's feature, will helpful to fix the > crash. > > This crash has been found in v4.17-rc1 using RaceFuzzer (a modified > version of Syzkaller), which we describe more at the end of this > report. Our analysis shows that the race occurs when invoking two > syscalls concurrently, write$rdma_cm and write$rdma_cm. Well, calling rdma_destroy_id() twice/concurrently is invalid.. The confusing part of this is how does it happen from ucma.c .. Double calls via write look OK to me, the ID is removed from the IDR at the top so it cannot be invoked twice.. So not sure what "write$rdma_cm and write$rdma_cm." is supposed to me? Is your test showing that write() vs close() is the problem? The oops suggests that.. And the logic around ctx->closing looks tortured enough that it is probably wrong... Jason
Re: KASAN: use-after-free Read in cma_cancel_operation
On Fri, May 11, 2018 at 02:25:22PM +0900, DaeRyong Jeong wrote: > We report the crash: KASAN: use-after-free Read in cma_cancel_operation > > Note that this bug is previously reported by syzkaller. > https://syzkaller.appspot.com/bug?id=95f89b8fb9fdc42e28ad586e657fea074e4e719b > Nonetheless, this bug has not fixed yet, and we hope that this report and our > analysis, which gets help by the RaceFuzzer's feature, will helpful to fix the > crash. > > This crash has been found in v4.17-rc1 using RaceFuzzer (a modified > version of Syzkaller), which we describe more at the end of this > report. Our analysis shows that the race occurs when invoking two > syscalls concurrently, write$rdma_cm and write$rdma_cm. Well, calling rdma_destroy_id() twice/concurrently is invalid.. The confusing part of this is how does it happen from ucma.c .. Double calls via write look OK to me, the ID is removed from the IDR at the top so it cannot be invoked twice.. So not sure what "write$rdma_cm and write$rdma_cm." is supposed to me? Is your test showing that write() vs close() is the problem? The oops suggests that.. And the logic around ctx->closing looks tortured enough that it is probably wrong... Jason
KASAN: use-after-free Read in cma_cancel_operation
We report the crash: KASAN: use-after-free Read in cma_cancel_operation Note that this bug is previously reported by syzkaller. https://syzkaller.appspot.com/bug?id=95f89b8fb9fdc42e28ad586e657fea074e4e719b Nonetheless, this bug has not fixed yet, and we hope that this report and our analysis, which gets help by the RaceFuzzer's feature, will helpful to fix the crash. This crash has been found in v4.17-rc1 using RaceFuzzer (a modified version of Syzkaller), which we describe more at the end of this report. Our analysis shows that the race occurs when invoking two syscalls concurrently, write$rdma_cm and write$rdma_cm. Analysis: We think the concurrent execution of rdma_destroy_id() causes the crash. The first execution of rdma_destroy_id() calls kfree(id_priv), and the second execution of rdma_destry_id() dereferences the id_priv in cma_cancel_listens(). Therefore use-after-free read occurs. We observed that rdma_destroy_id() is called during the write$rdma_cm syscall. After returing from vfs_write(), fput() is called and ucma_close() is called as a pending work before returing to the user space. Thread interleaving: CPU0 (rdma_destory_id) CPU1 (rdma_destroy_id) = = kfree(id_priv->id.route.path_rec); put_net(id_priv->id.route.addr.dev_addr.net); kfree(id_priv); id_priv = container_of(id, struct rdma_id_private, id); state = cma_exch(id_priv, RDMA_CM_DESTROYING); cma_cancel_operation(id_priv, state); (in cma_cancel_listens) list_del(_priv->list); Call Sequence: Both CPU0 and CPU1 = ucma_close rdma_destroy_id == BUG: KASAN: use-after-free in __list_del_entry_valid+0x5c/0xc0 lib/list_debug.c:54 Read of size 8 at addr 8801e86deca0 by task syz-executor0/3524 CPU: 1 PID: 3524 Comm: syz-executor0 Not tainted 4.17.0-rc1 #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.8.2-0-g33fbe13 by qemu-project.org 04/01/2014 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x166/0x21c lib/dump_stack.c:113 print_address_description+0x73/0x250 mm/kasan/report.c:256 kasan_report_error mm/kasan/report.c:354 [inline] kasan_report+0x23f/0x360 mm/kasan/report.c:412 check_memory_region_inline mm/kasan/kasan.c:260 [inline] __asan_load8+0x54/0x90 mm/kasan/kasan.c:699 __list_del_entry_valid+0x5c/0xc0 lib/list_debug.c:54 __list_del_entry include/linux/list.h:117 [inline] list_del include/linux/list.h:125 [inline] cma_cancel_listens drivers/infiniband/core/cma.c:1527 [inline] cma_cancel_operation+0x2d2/0x750 drivers/infiniband/core/cma.c:1555 rdma_destroy_id+0xe9/0x760 drivers/infiniband/core/cma.c:1619 ucma_close+0x9f/0x1c0 drivers/infiniband/core/ucma.c:1743 __fput+0x22c/0x450 fs/file_table.c:209 fput+0x15/0x20 fs/file_table.c:243 task_work_run+0x152/0x1b0 kernel/task_work.c:113 exit_task_work include/linux/task_work.h:22 [inline] do_exit+0x1387/0x1860 kernel/exit.c:865 do_group_exit+0xfb/0x220 kernel/exit.c:968 get_signal+0x5b7/0xf70 kernel/signal.c:2469 do_signal+0x94/0xde0 arch/x86/kernel/signal.c:810 exit_to_usermode_loop+0x1eb/0x270 arch/x86/entry/common.c:162 prepare_exit_to_usermode arch/x86/entry/common.c:196 [inline] syscall_return_slowpath arch/x86/entry/common.c:265 [inline] do_syscall_64+0x473/0x4a0 arch/x86/entry/common.c:290 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x4563f9 RSP: 002b:7fdd885d9ba8 EFLAGS: 0246 ORIG_RAX: 00ca RAX: fe00 RBX: 0072bfc8 RCX: 004563f9 RDX: RSI: RDI: 0072bfc8 RBP: 7fdd885d9bd0 R08: R09: 0072bfa0 R10: R11: 0246 R12: 0001 R13: 7fdd885d9c50 R14: R15: 7fdd885da700 Allocated by task 3521: save_stack+0x43/0xd0 mm/kasan/kasan.c:448 set_track mm/kasan/kasan.c:460 [inline] kasan_kmalloc+0xae/0xe0 mm/kasan/kasan.c:553 kmem_cache_alloc_trace+0x136/0x740 mm/slab.c:3620 kmalloc include/linux/slab.h:512 [inline] kzalloc include/linux/slab.h:701 [inline] __rdma_create_id+0xc5/0x450 drivers/infiniband/core/cma.c:751 ucma_create_id+0x219/0x510 drivers/infiniband/core/ucma.c:485 ucma_write+0x1d6/0x260 drivers/infiniband/core/ucma.c:1664 __vfs_write+0xdd/0x480 fs/read_write.c:485 vfs_write+0x12d/0x2d0 fs/read_write.c:549 ksys_write+0xca/0x190 fs/read_write.c:598 __do_sys_write fs/read_write.c:610 [inline] __se_sys_write fs/read_write.c:607 [inline] __x64_sys_write+0x43/0x50 fs/read_write.c:607 do_syscall_64+0x15f/0x4a0 arch/x86/entry/common.c:287 entry_SYSCALL_64_after_hwf
KASAN: use-after-free Read in cma_cancel_operation
We report the crash: KASAN: use-after-free Read in cma_cancel_operation Note that this bug is previously reported by syzkaller. https://syzkaller.appspot.com/bug?id=95f89b8fb9fdc42e28ad586e657fea074e4e719b Nonetheless, this bug has not fixed yet, and we hope that this report and our analysis, which gets help by the RaceFuzzer's feature, will helpful to fix the crash. This crash has been found in v4.17-rc1 using RaceFuzzer (a modified version of Syzkaller), which we describe more at the end of this report. Our analysis shows that the race occurs when invoking two syscalls concurrently, write$rdma_cm and write$rdma_cm. Analysis: We think the concurrent execution of rdma_destroy_id() causes the crash. The first execution of rdma_destroy_id() calls kfree(id_priv), and the second execution of rdma_destry_id() dereferences the id_priv in cma_cancel_listens(). Therefore use-after-free read occurs. We observed that rdma_destroy_id() is called during the write$rdma_cm syscall. After returing from vfs_write(), fput() is called and ucma_close() is called as a pending work before returing to the user space. Thread interleaving: CPU0 (rdma_destory_id) CPU1 (rdma_destroy_id) = = kfree(id_priv->id.route.path_rec); put_net(id_priv->id.route.addr.dev_addr.net); kfree(id_priv); id_priv = container_of(id, struct rdma_id_private, id); state = cma_exch(id_priv, RDMA_CM_DESTROYING); cma_cancel_operation(id_priv, state); (in cma_cancel_listens) list_del(_priv->list); Call Sequence: Both CPU0 and CPU1 = ucma_close rdma_destroy_id == BUG: KASAN: use-after-free in __list_del_entry_valid+0x5c/0xc0 lib/list_debug.c:54 Read of size 8 at addr 8801e86deca0 by task syz-executor0/3524 CPU: 1 PID: 3524 Comm: syz-executor0 Not tainted 4.17.0-rc1 #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.8.2-0-g33fbe13 by qemu-project.org 04/01/2014 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x166/0x21c lib/dump_stack.c:113 print_address_description+0x73/0x250 mm/kasan/report.c:256 kasan_report_error mm/kasan/report.c:354 [inline] kasan_report+0x23f/0x360 mm/kasan/report.c:412 check_memory_region_inline mm/kasan/kasan.c:260 [inline] __asan_load8+0x54/0x90 mm/kasan/kasan.c:699 __list_del_entry_valid+0x5c/0xc0 lib/list_debug.c:54 __list_del_entry include/linux/list.h:117 [inline] list_del include/linux/list.h:125 [inline] cma_cancel_listens drivers/infiniband/core/cma.c:1527 [inline] cma_cancel_operation+0x2d2/0x750 drivers/infiniband/core/cma.c:1555 rdma_destroy_id+0xe9/0x760 drivers/infiniband/core/cma.c:1619 ucma_close+0x9f/0x1c0 drivers/infiniband/core/ucma.c:1743 __fput+0x22c/0x450 fs/file_table.c:209 fput+0x15/0x20 fs/file_table.c:243 task_work_run+0x152/0x1b0 kernel/task_work.c:113 exit_task_work include/linux/task_work.h:22 [inline] do_exit+0x1387/0x1860 kernel/exit.c:865 do_group_exit+0xfb/0x220 kernel/exit.c:968 get_signal+0x5b7/0xf70 kernel/signal.c:2469 do_signal+0x94/0xde0 arch/x86/kernel/signal.c:810 exit_to_usermode_loop+0x1eb/0x270 arch/x86/entry/common.c:162 prepare_exit_to_usermode arch/x86/entry/common.c:196 [inline] syscall_return_slowpath arch/x86/entry/common.c:265 [inline] do_syscall_64+0x473/0x4a0 arch/x86/entry/common.c:290 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x4563f9 RSP: 002b:7fdd885d9ba8 EFLAGS: 0246 ORIG_RAX: 00ca RAX: fe00 RBX: 0072bfc8 RCX: 004563f9 RDX: RSI: RDI: 0072bfc8 RBP: 7fdd885d9bd0 R08: R09: 0072bfa0 R10: R11: 0246 R12: 0001 R13: 7fdd885d9c50 R14: R15: 7fdd885da700 Allocated by task 3521: save_stack+0x43/0xd0 mm/kasan/kasan.c:448 set_track mm/kasan/kasan.c:460 [inline] kasan_kmalloc+0xae/0xe0 mm/kasan/kasan.c:553 kmem_cache_alloc_trace+0x136/0x740 mm/slab.c:3620 kmalloc include/linux/slab.h:512 [inline] kzalloc include/linux/slab.h:701 [inline] __rdma_create_id+0xc5/0x450 drivers/infiniband/core/cma.c:751 ucma_create_id+0x219/0x510 drivers/infiniband/core/ucma.c:485 ucma_write+0x1d6/0x260 drivers/infiniband/core/ucma.c:1664 __vfs_write+0xdd/0x480 fs/read_write.c:485 vfs_write+0x12d/0x2d0 fs/read_write.c:549 ksys_write+0xca/0x190 fs/read_write.c:598 __do_sys_write fs/read_write.c:610 [inline] __se_sys_write fs/read_write.c:607 [inline] __x64_sys_write+0x43/0x50 fs/read_write.c:607 do_syscall_64+0x15f/0x4a0 arch/x86/entry/common.c:287 entry_SYSCALL_64_after_hwf
KASAN: use-after-free Read in cma_cancel_operation
Hello, syzbot hit the following crash on upstream commit 86bbbebac1933e6e95e8234c4f7d220c5ddd38bc (Mon Apr 2 18:47:07 2018 +) Merge branch 'ras-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip syzbot dashboard link: https://syzkaller.appspot.com/bug?extid=6956235342b7317ec564 So far this crash happened 4 times on upstream. C reproducer: https://syzkaller.appspot.com/x/repro.c?id=6621340819783680 syzkaller reproducer: https://syzkaller.appspot.com/x/repro.syz?id=6075521074987008 Raw console output: https://syzkaller.appspot.com/x/log.txt?id=6447569127342080 Kernel config: https://syzkaller.appspot.com/x/.config?id=6801295859785128502 compiler: gcc (GCC) 7.1.1 20170620 IMPORTANT: if you fix the bug, please add the following tag to the commit: Reported-by: syzbot+6956235342b7317ec...@syzkaller.appspotmail.com It will help syzbot understand when the bug is fixed. See footer for details. If you forward the report, please keep this part and the footer. == BUG: KASAN: use-after-free in __list_del_entry_valid+0x144/0x150 lib/list_debug.c:54 Read of size 8 at addr 8801d8dc7d20 by task syzkaller468166/4496 CPU: 0 PID: 4496 Comm: syzkaller468166 Not tainted 4.16.0+ #376 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:17 [inline] dump_stack+0x1a7/0x27d lib/dump_stack.c:53 print_address_description+0x73/0x250 mm/kasan/report.c:256 kasan_report_error mm/kasan/report.c:354 [inline] kasan_report+0x23c/0x360 mm/kasan/report.c:412 __asan_report_load8_noabort+0x14/0x20 mm/kasan/report.c:433 __list_del_entry_valid+0x144/0x150 lib/list_debug.c:54 __list_del_entry include/linux/list.h:117 [inline] list_del include/linux/list.h:125 [inline] cma_cancel_listens drivers/infiniband/core/cma.c:1569 [inline] cma_cancel_operation+0x455/0xd60 drivers/infiniband/core/cma.c:1597 rdma_destroy_id+0xff/0xda0 drivers/infiniband/core/cma.c:1661 ucma_close+0x100/0x2f0 drivers/infiniband/core/ucma.c:1735 __fput+0x327/0x7f0 fs/file_table.c:209 fput+0x15/0x20 fs/file_table.c:243 task_work_run+0x1ab/0x280 kernel/task_work.c:113 exit_task_work include/linux/task_work.h:22 [inline] do_exit+0x1986/0x2700 kernel/exit.c:865 do_group_exit+0x149/0x400 kernel/exit.c:968 get_signal+0x74c/0x16e0 kernel/signal.c:2469 do_signal+0x90/0x1e90 arch/x86/kernel/signal.c:809 exit_to_usermode_loop+0x258/0x2f0 arch/x86/entry/common.c:162 prepare_exit_to_usermode arch/x86/entry/common.c:196 [inline] syscall_return_slowpath arch/x86/entry/common.c:265 [inline] do_syscall_64+0x6ec/0x940 arch/x86/entry/common.c:292 entry_SYSCALL_64_after_hwframe+0x42/0xb7 RIP: 0033:0x446b09 RSP: 002b:7f2b89150da8 EFLAGS: 0246 ORIG_RAX: 00ca RAX: fe00 RBX: 006e29fc RCX: 00446b09 RDX: RSI: RDI: 006e29fc RBP: 006e29f8 R08: R09: R10: R11: 0246 R12: 006d635f616d6472 R13: 2f646e6162696e69 R14: 666e692f7665642f R15: 0005 Allocated by task 4493: save_stack+0x43/0xd0 mm/kasan/kasan.c:447 set_track mm/kasan/kasan.c:459 [inline] kasan_kmalloc+0xad/0xe0 mm/kasan/kasan.c:552 kmem_cache_alloc_trace+0x136/0x740 mm/slab.c:3608 kmalloc include/linux/slab.h:512 [inline] kzalloc include/linux/slab.h:701 [inline] rdma_create_id+0xd0/0x640 drivers/infiniband/core/cma.c:787 ucma_create_id+0x35f/0x920 drivers/infiniband/core/ucma.c:480 ucma_write+0x2d6/0x3d0 drivers/infiniband/core/ucma.c:1656 __vfs_write+0xef/0x970 fs/read_write.c:480 vfs_write+0x189/0x510 fs/read_write.c:544 SYSC_write fs/read_write.c:589 [inline] SyS_write+0xef/0x220 fs/read_write.c:581 do_syscall_64+0x281/0x940 arch/x86/entry/common.c:287 entry_SYSCALL_64_after_hwframe+0x42/0xb7 Freed by task 4496: save_stack+0x43/0xd0 mm/kasan/kasan.c:447 set_track mm/kasan/kasan.c:459 [inline] __kasan_slab_free+0x11a/0x170 mm/kasan/kasan.c:520 kasan_slab_free+0xe/0x10 mm/kasan/kasan.c:527 __cache_free mm/slab.c:3486 [inline] kfree+0xd9/0x260 mm/slab.c:3801 rdma_destroy_id+0x821/0xda0 drivers/infiniband/core/cma.c:1691 ucma_close+0x100/0x2f0 drivers/infiniband/core/ucma.c:1735 __fput+0x327/0x7f0 fs/file_table.c:209 fput+0x15/0x20 fs/file_table.c:243 task_work_run+0x1ab/0x280 kernel/task_work.c:113 exit_task_work include/linux/task_work.h:22 [inline] do_exit+0x1986/0x2700 kernel/exit.c:865 do_group_exit+0x149/0x400 kernel/exit.c:968 get_signal+0x74c/0x16e0 kernel/signal.c:2469 do_signal+0x90/0x1e90 arch/x86/kernel/signal.c:809 exit_to_usermode_loop+0x258/0x2f0 arch/x86/entry/common.c:162 prepare_exit_to_usermode arch/x86/entry/common.c:196 [inline] syscall_return_slowpath arch/x86/entry/common.c:265 [inline] do_syscall_64+0x6ec/0x940 arch/x86/entry/common.c:292 entry_SYSCALL_64_after_hwframe+0x42/0xb7 The buggy
KASAN: use-after-free Read in cma_cancel_operation
Hello, syzbot hit the following crash on upstream commit 86bbbebac1933e6e95e8234c4f7d220c5ddd38bc (Mon Apr 2 18:47:07 2018 +) Merge branch 'ras-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip syzbot dashboard link: https://syzkaller.appspot.com/bug?extid=6956235342b7317ec564 So far this crash happened 4 times on upstream. C reproducer: https://syzkaller.appspot.com/x/repro.c?id=6621340819783680 syzkaller reproducer: https://syzkaller.appspot.com/x/repro.syz?id=6075521074987008 Raw console output: https://syzkaller.appspot.com/x/log.txt?id=6447569127342080 Kernel config: https://syzkaller.appspot.com/x/.config?id=6801295859785128502 compiler: gcc (GCC) 7.1.1 20170620 IMPORTANT: if you fix the bug, please add the following tag to the commit: Reported-by: syzbot+6956235342b7317ec...@syzkaller.appspotmail.com It will help syzbot understand when the bug is fixed. See footer for details. If you forward the report, please keep this part and the footer. == BUG: KASAN: use-after-free in __list_del_entry_valid+0x144/0x150 lib/list_debug.c:54 Read of size 8 at addr 8801d8dc7d20 by task syzkaller468166/4496 CPU: 0 PID: 4496 Comm: syzkaller468166 Not tainted 4.16.0+ #376 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:17 [inline] dump_stack+0x1a7/0x27d lib/dump_stack.c:53 print_address_description+0x73/0x250 mm/kasan/report.c:256 kasan_report_error mm/kasan/report.c:354 [inline] kasan_report+0x23c/0x360 mm/kasan/report.c:412 __asan_report_load8_noabort+0x14/0x20 mm/kasan/report.c:433 __list_del_entry_valid+0x144/0x150 lib/list_debug.c:54 __list_del_entry include/linux/list.h:117 [inline] list_del include/linux/list.h:125 [inline] cma_cancel_listens drivers/infiniband/core/cma.c:1569 [inline] cma_cancel_operation+0x455/0xd60 drivers/infiniband/core/cma.c:1597 rdma_destroy_id+0xff/0xda0 drivers/infiniband/core/cma.c:1661 ucma_close+0x100/0x2f0 drivers/infiniband/core/ucma.c:1735 __fput+0x327/0x7f0 fs/file_table.c:209 fput+0x15/0x20 fs/file_table.c:243 task_work_run+0x1ab/0x280 kernel/task_work.c:113 exit_task_work include/linux/task_work.h:22 [inline] do_exit+0x1986/0x2700 kernel/exit.c:865 do_group_exit+0x149/0x400 kernel/exit.c:968 get_signal+0x74c/0x16e0 kernel/signal.c:2469 do_signal+0x90/0x1e90 arch/x86/kernel/signal.c:809 exit_to_usermode_loop+0x258/0x2f0 arch/x86/entry/common.c:162 prepare_exit_to_usermode arch/x86/entry/common.c:196 [inline] syscall_return_slowpath arch/x86/entry/common.c:265 [inline] do_syscall_64+0x6ec/0x940 arch/x86/entry/common.c:292 entry_SYSCALL_64_after_hwframe+0x42/0xb7 RIP: 0033:0x446b09 RSP: 002b:7f2b89150da8 EFLAGS: 0246 ORIG_RAX: 00ca RAX: fe00 RBX: 006e29fc RCX: 00446b09 RDX: RSI: RDI: 006e29fc RBP: 006e29f8 R08: R09: R10: R11: 0246 R12: 006d635f616d6472 R13: 2f646e6162696e69 R14: 666e692f7665642f R15: 0005 Allocated by task 4493: save_stack+0x43/0xd0 mm/kasan/kasan.c:447 set_track mm/kasan/kasan.c:459 [inline] kasan_kmalloc+0xad/0xe0 mm/kasan/kasan.c:552 kmem_cache_alloc_trace+0x136/0x740 mm/slab.c:3608 kmalloc include/linux/slab.h:512 [inline] kzalloc include/linux/slab.h:701 [inline] rdma_create_id+0xd0/0x640 drivers/infiniband/core/cma.c:787 ucma_create_id+0x35f/0x920 drivers/infiniband/core/ucma.c:480 ucma_write+0x2d6/0x3d0 drivers/infiniband/core/ucma.c:1656 __vfs_write+0xef/0x970 fs/read_write.c:480 vfs_write+0x189/0x510 fs/read_write.c:544 SYSC_write fs/read_write.c:589 [inline] SyS_write+0xef/0x220 fs/read_write.c:581 do_syscall_64+0x281/0x940 arch/x86/entry/common.c:287 entry_SYSCALL_64_after_hwframe+0x42/0xb7 Freed by task 4496: save_stack+0x43/0xd0 mm/kasan/kasan.c:447 set_track mm/kasan/kasan.c:459 [inline] __kasan_slab_free+0x11a/0x170 mm/kasan/kasan.c:520 kasan_slab_free+0xe/0x10 mm/kasan/kasan.c:527 __cache_free mm/slab.c:3486 [inline] kfree+0xd9/0x260 mm/slab.c:3801 rdma_destroy_id+0x821/0xda0 drivers/infiniband/core/cma.c:1691 ucma_close+0x100/0x2f0 drivers/infiniband/core/ucma.c:1735 __fput+0x327/0x7f0 fs/file_table.c:209 fput+0x15/0x20 fs/file_table.c:243 task_work_run+0x1ab/0x280 kernel/task_work.c:113 exit_task_work include/linux/task_work.h:22 [inline] do_exit+0x1986/0x2700 kernel/exit.c:865 do_group_exit+0x149/0x400 kernel/exit.c:968 get_signal+0x74c/0x16e0 kernel/signal.c:2469 do_signal+0x90/0x1e90 arch/x86/kernel/signal.c:809 exit_to_usermode_loop+0x258/0x2f0 arch/x86/entry/common.c:162 prepare_exit_to_usermode arch/x86/entry/common.c:196 [inline] syscall_return_slowpath arch/x86/entry/common.c:265 [inline] do_syscall_64+0x6ec/0x940 arch/x86/entry/common.c:292 entry_SYSCALL_64_after_hwframe+0x42/0xb7 The buggy
