kprobe.multi programs run in atomic/RCU context and cannot sleep.
However, bpf_kprobe_multi_link_attach() did not validate whether the
program being attached had the sleepable flag set, allowing sleepable
helpers such as bpf_copy_from_user() to be invoked from a non-sleepable
context.
This causes a "sleeping function called from invalid context" splat:
BUG: sleeping function called from invalid context at
./include/linux/uaccess.h:169
in_atomic(): 1, irqs_disabled(): 0, non_block: 0, pid: 1787, name: sudo
preempt_count: 1, expected: 0
RCU nest depth: 2, expected: 0
Fix this by rejecting sleepable programs early in
bpf_kprobe_multi_link_attach(), before any further processing.
Also add a selftest that tries to load a sleepable kprobe_multi program
and it needs to be rejected for the test to pass.
Fixes: 0dcac2725406 ("bpf: Add multi kprobe link")
Signed-off-by: Varun R Mallya <[email protected]>
---
kernel/trace/bpf_trace.c | 4 ++
.../bpf/prog_tests/kprobe_multi_test.c | 41 +++++++++++++++++++
.../bpf/progs/kprobe_multi_sleepable.c | 13 ++++++
3 files changed, 58 insertions(+)
create mode 100644 tools/testing/selftests/bpf/progs/kprobe_multi_sleepable.c
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 0b040a417442..af7079aa0f36 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -2752,6 +2752,10 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr
*attr, struct bpf_prog *pr
if (!is_kprobe_multi(prog))
return -EINVAL;
+ /* kprobe_multi is not allowed to be sleepable. */
+ if (prog->sleepable)
+ return -EINVAL;
+
/* Writing to context is not allowed for kprobes. */
if (prog->aux->kprobe_write_ctx)
return -EINVAL;
diff --git a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c
b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c
index 78c974d4ea33..f02fec2b6fda 100644
--- a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c
+++ b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c
@@ -10,6 +10,7 @@
#include "kprobe_multi_session_cookie.skel.h"
#include "kprobe_multi_verifier.skel.h"
#include "kprobe_write_ctx.skel.h"
+#include "kprobe_multi_sleepable.skel.h"
#include "bpf/libbpf_internal.h"
#include "bpf/hashmap.h"
@@ -633,6 +634,44 @@ static void test_attach_write_ctx(void)
}
#endif
+static void test_attach_multi_sleepable(void)
+{
+ struct kprobe_multi_sleepable *skel;
+ int err;
+
+ skel = kprobe_multi_sleepable__open();
+ if (!ASSERT_OK_PTR(skel, "kprobe_multi_sleepable__open"))
+ return;
+
+ err = bpf_program__set_flags(skel->progs.handle_kprobe_multi_sleepable,
+ BPF_F_SLEEPABLE);
+ if (!ASSERT_OK(err, "bpf_program__set_flags"))
+ goto cleanup;
+
+ /* Load should succeed even with BPF_F_SLEEPABLE for KPROBE types */
+ err = kprobe_multi_sleepable__load(skel);
+ if (!ASSERT_OK(err, "kprobe_multi_sleepable__load"))
+ goto cleanup;
+
+ /* Attachment must fail for kprobe.multi + BPF_F_SLEEPABLE.
+ * Also chosen a stable symbol to send into opts
+ */
+ LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
+ const char *sym = "vfs_read";
+
+ opts.syms = &sym;
+ opts.cnt = 1;
+
+ skel->links.handle_kprobe_multi_sleepable =
+
bpf_program__attach_kprobe_multi_opts(skel->progs.handle_kprobe_multi_sleepable,
+ NULL, &opts);
+ ASSERT_ERR_PTR(skel->links.handle_kprobe_multi_sleepable,
+ "bpf_program__attach_kprobe_multi_opts");
+
+cleanup:
+ kprobe_multi_sleepable__destroy(skel);
+}
+
void serial_test_kprobe_multi_bench_attach(void)
{
if (test__start_subtest("kernel"))
@@ -676,5 +715,7 @@ void test_kprobe_multi_test(void)
test_unique_match();
if (test__start_subtest("attach_write_ctx"))
test_attach_write_ctx();
+ if (test__start_subtest("attach_multi_sleepable"))
+ test_attach_multi_sleepable();
RUN_TESTS(kprobe_multi_verifier);
}
diff --git a/tools/testing/selftests/bpf/progs/kprobe_multi_sleepable.c
b/tools/testing/selftests/bpf/progs/kprobe_multi_sleepable.c
new file mode 100644
index 000000000000..56973ad8779d
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/kprobe_multi_sleepable.c
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+SEC("kprobe.multi")
+int handle_kprobe_multi_sleepable(struct pt_regs *ctx)
+{
+ return 0;
+}
+
+char _license[] SEC("license") = "GPL";
--
2.53.0
