From: Masami Hiramatsu (Google) <[email protected]>
Several edge cases in the optprobe state machine can prematurely release
instruction slots or stall blocked sibling probes:
1. In kill_optimized_kprobe(), when a probed module is unloaded while child
probes still exist on p->list, the probe is not unused (!kprobe_unused(p)).
If op was queued for unoptimization, move it to the cooling list rather
than immediately calling arch_remove_optimized_kprobe(), ensuring Tasks
RCU elapses before slot reclamation. In optprobe_finalize_generation(),
reclaim the optinsn slot once cooling finishes for kprobe_gone() probes.
2. In optprobe_finalize_generation(), when a disabled cooling probe
finishes cooling and transitions to safely disarmed, re-optimize any
blocked sibling probe at the same address.
Fixes: 06e0d3b89da1 ("kprobes: Make optprobe optimizer multi-generational and
asynchronous")
Signed-off-by: Masami Hiramatsu (Google) <[email protected]>
---
kernel/kprobes.c | 22 +++++++++++++++++++---
1 file changed, 19 insertions(+), 3 deletions(-)
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 0ef3a8701eda..53725c09ef9b 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -725,10 +725,19 @@ static void optprobe_finalize_generation(struct
optprobe_generation *gen)
list_move(&op->list,
&optprobe_gens[optprobe_cur_gen].freeing_list);
kick_kprobe_optimizer();
} else {
+ struct kprobe *_p;
+
/* Still in use; now safely disarmed */
list_del_init(&op->list);
- if (!kprobe_disabled(&op->kp))
+ if (kprobe_gone(&op->kp)) {
+ arch_remove_optimized_kprobe(op);
+ } else if (!kprobe_disabled(&op->kp)) {
optimize_kprobe(&op->kp);
+ } else {
+ _p = get_optimized_kprobe(op->kp.addr);
+ if (unlikely(_p))
+ optimize_kprobe(_p);
+ }
}
}
@@ -1084,13 +1093,20 @@ static void kill_optimized_kprobe(struct kprobe *p)
list_del_init(&op->list);
}
} else {
- list_del_init(&op->list);
+ if (optprobe_queued_unopt(op)) {
+ /* Move to cooling list so optinsn slot is
cooled down */
+ list_move(&op->list,
+
&optprobe_gens[optprobe_cur_gen].cooling_list);
+ kick_kprobe_optimizer();
+ } else if (!optprobe_queued_cooling(op) &&
!optprobe_queued_freeing(op)) {
+ list_del_init(&op->list);
+ }
}
}
op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
/*
- * Don't remove the slot if it is queued for freeing or unoptimization;
+ * Don't remove the slot if it is queued for freeing, unoptimization,
or cooling;
* the optimizer will reclaim it after the quiescence period.
*/
if (!optprobe_queued_freeing(op) && !optprobe_queued_unopt(op) &&
