Exit single-stepping out of line and get back regs->ip to original
(probed) address before trying fixup_exception() if the exception
happened on the singlestep buffer, since the fixup_exception()
depends on regs->ip to search an entry on __ex_table.
Signed-off-by: Masami Hiramatsu <mhira...@kernel.org>
---
arch/x86/include/asm/kprobes.h | 1
arch/x86/kernel/kprobes/core.c | 83 +++++++++++++++++++++++++---------------
arch/x86/kernel/traps.c | 19 +++++++++
3 files changed, 71 insertions(+), 32 deletions(-)
diff --git a/arch/x86/include/asm/kprobes.h b/arch/x86/include/asm/kprobes.h
index d1d1e50..79e121a 100644
--- a/arch/x86/include/asm/kprobes.h
+++ b/arch/x86/include/asm/kprobes.h
@@ -111,6 +111,7 @@ struct kprobe_ctlblk {
struct prev_kprobe prev_kprobe;
};
+extern int kprobe_exit_singlestep(struct pt_regs *regs);
extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr);
extern int kprobe_exceptions_notify(struct notifier_block *self,
unsigned long val, void *data);
diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c
index 34d3a52..f2a3f3b 100644
--- a/arch/x86/kernel/kprobes/core.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -949,43 +949,62 @@ int kprobe_debug_handler(struct pt_regs *regs)
}
NOKPROBE_SYMBOL(kprobe_debug_handler);
-int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
+/* Fixup current ip register and reset current kprobe, if needed. */
+int kprobe_exit_singlestep(struct pt_regs *regs)
{
- struct kprobe *cur = kprobe_running();
struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
+ struct kprobe *cur = kprobe_running();
- if (unlikely(regs->ip == (unsigned long)cur->ainsn.insn)) {
- /* This must happen on single-stepping */
- WARN_ON(kcb->kprobe_status != KPROBE_HIT_SS &&
- kcb->kprobe_status != KPROBE_REENTER);
- /*
- * We are here because the instruction being single
- * stepped caused a page fault. We reset the current
- * kprobe and the ip points back to the probe address
- * and allow the page fault handler to continue as a
- * normal page fault.
- */
- regs->ip = (unsigned long)cur->addr;
- /*
- * Trap flag (TF) has been set here because this fault
- * happened where the single stepping will be done.
- * So clear it by resetting the current kprobe:
- */
- regs->flags &= ~X86_EFLAGS_TF;
+ if (unlikely(regs->ip != (unsigned long)cur->ainsn.insn))
+ return 0;
- /*
- * If the TF flag was set before the kprobe hit,
- * don't touch it:
- */
- regs->flags |= kcb->kprobe_old_flags;
+ /* This must happen on single-stepping */
+ WARN_ON(kcb->kprobe_status != KPROBE_HIT_SS &&
+ kcb->kprobe_status != KPROBE_REENTER);
+ /*
+ * We are here because the instruction being single
+ * stepped caused a page fault. We reset the current
+ * kprobe and the ip points back to the probe address
+ * and allow the page fault handler to continue as a
+ * normal page fault.
+ */
+ regs->ip = (unsigned long)cur->addr;
+ /*
+ * Trap flag (TF) has been set here because this fault
+ * happened where the single stepping will be done.
+ * So clear it by resetting the current kprobe:
+ */
+ regs->flags &= ~X86_EFLAGS_TF;
- if (kcb->kprobe_status == KPROBE_REENTER)
- restore_previous_kprobe(kcb);
- else
- reset_current_kprobe();
- preempt_enable_no_resched();
- } else if (kcb->kprobe_status == KPROBE_HIT_ACTIVE ||
- kcb->kprobe_status == KPROBE_HIT_SSDONE) {
+ /*
+ * If the TF flag was set before the kprobe hit,
+ * don't touch it:
+ */
+ regs->flags |= kcb->kprobe_old_flags;
+
+ if (kcb->kprobe_status == KPROBE_REENTER)
+ restore_previous_kprobe(kcb);
+ else
+ reset_current_kprobe();
+
+ /* Preempt has been disabled before single stepping */
+ preempt_enable_no_resched();
+
+ return 1;
+}
+NOKPROBE_SYMBOL(kprobe_exit_singlestep);
+
+int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
+{
+ struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
+ struct kprobe *cur = kprobe_running();
+
+ /* If the fault happened on singlestep, finish it and retry */
+ if (kprobe_exit_singlestep(regs))
+ return 0;
+
+ if (kcb->kprobe_status == KPROBE_HIT_ACTIVE ||
+ kcb->kprobe_status == KPROBE_HIT_SSDONE) {
/*
* We increment the nmissed count for accounting,
* we can also use npre/npostfault count for accounting
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 948443e..7ac1baf 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -187,6 +187,14 @@ do_trap_no_signal(struct task_struct *tsk, int trapnr,
char *str,
}
if (!user_mode(regs)) {
+ /*
+ * Exit from the kprobe's single-stepping before trying
+ * fixup_exception() because the fixup routine is based on
+ * trapped address (regs->ip). Single-stepping out of line
+ * executes an instruction in different place, so it should
+ * be fixed.
+ */
+ kprobe_exit_singlestep(regs);
if (!fixup_exception(regs, trapnr)) {
tsk->thread.error_code = error_code;
tsk->thread.trap_nr = trapnr;
@@ -500,6 +508,12 @@ do_general_protection(struct pt_regs *regs, long
error_code)
tsk = current;
if (!user_mode(regs)) {
+ /*
+ * Exit from the kprobe's single-stepping before trying
+ * fixup_exception(). Note that if the GPF occurred in
+ * kprobe user handlers, it is handled in notify_die.
+ */
+ kprobe_exit_singlestep(regs);
if (fixup_exception(regs, X86_TRAP_GP))
return;
@@ -799,6 +813,11 @@ static void math_error(struct pt_regs *regs, int
error_code, int trapnr)
cond_local_irq_enable(regs);
if (!user_mode(regs)) {
+ /*
+ * Exit from the kprobe's single-stepping before trying
+ * fixup_exception().
+ */
+ kprobe_exit_singlestep(regs);
if (!fixup_exception(regs, trapnr)) {
task->thread.error_code = error_code;
task->thread.trap_nr = trapnr;
