early_fixup_exception is limited by the fact that it doesn't have a
real struct pt_regs.  Change both the 32-bit and 64-bit asm and the
C code to pass and accept a real pt_regs.

Signed-off-by: Andy Lutomirski <l...@kernel.org>
---
 arch/x86/include/asm/uaccess.h |  2 +-
 arch/x86/kernel/head_32.S      | 74 +++++++++++++++++++++++++++++-------------
 arch/x86/kernel/head_64.S      | 68 ++++++++++++++++++++------------------
 arch/x86/mm/extable.c          |  6 ++--
 4 files changed, 92 insertions(+), 58 deletions(-)

diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index c0f27d7ea7ff..a3afb7259751 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -111,7 +111,7 @@ struct exception_table_entry {
 
 extern int fixup_exception(struct pt_regs *regs, int trapnr);
 extern bool ex_has_fault_handler(unsigned long ip);
-extern int early_fixup_exception(unsigned long *ip);
+extern int early_fixup_exception(struct pt_regs *regs, int trapnr);
 
 /*
  * These are the main single-value transfer routines.  They automatically
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 6bc9ae24b6d2..bef3e6c49b56 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -562,29 +562,64 @@ early_idt_handler_common:
        je hlt_loop
        incl %ss:early_recursion_flag
 
-       push %eax               # 16(%esp)
-       push %ecx               # 12(%esp)
-       push %edx               #  8(%esp)
-       push %ds                #  4(%esp)
-       push %es                #  0(%esp)
-       movl $(__KERNEL_DS),%eax
-       movl %eax,%ds
-       movl %eax,%es
+       /* The vector number is in pt_regs->gs */
 
-       cmpl $(__KERNEL_CS),32(%esp)
+       cld
+       pushl   %fs             /* pt_regs->fs */
+       movw    $0, 2(%esp)     /* clear high bits (some CPUs leave garbage) */
+       pushl   %es             /* pt_regs->es */
+       movw    $0, 2(%esp)     /* clear high bits (some CPUs leave garbage) */
+       pushl   %ds             /* pt_regs->ds */
+       movw    $0, 2(%esp)     /* clear high bits (some CPUs leave garbage) */
+       pushl   %eax            /* pt_regs->ax */
+       pushl   %ebp            /* pt_regs->bp */
+       pushl   %edi            /* pt_regs->di */
+       pushl   %esi            /* pt_regs->si */
+       pushl   %edx            /* pt_regs->dx */
+       pushl   %ecx            /* pt_regs->cx */
+       pushl   %ebx            /* pt_regs->bx */
+
+       /* Fix up DS and ES */
+       movl    $(__KERNEL_DS), %ecx
+       movl    %ecx, %ds
+       movl    %ecx, %es
+
+       /* Load the vector number into EDX */
+       movl    PT_GS(%esp), %edx
+
+       /* Load GS into pt_regs->gs and clear high bits */
+       movw    %gs, PT_GS(%esp)
+       movw    $0, PT_GS+2(%esp)
+
+       cmpl $(__KERNEL_CS),PT_CS(%esp)
        jne 10f
 
-       leal 28(%esp),%eax      # Pointer to %eip
-       call early_fixup_exception
-       andl %eax,%eax
-       jnz ex_entry            /* found an exception entry */
+       movl    %esp, %eax      /* args are pt_regs (EAX), trapnr (EDX) */
+       call    early_fixup_exception
+       andl    %eax,%eax
+       jz      10f             /* Exception wasn't fixed up */
+
+       popl    %ebx            /* pt_regs->bx */
+       popl    %ecx            /* pt_regs->cx */
+       popl    %edx            /* pt_regs->dx */
+       popl    %esi            /* pt_regs->si */
+       popl    %edi            /* pt_regs->di */
+       popl    %ebp            /* pt_regs->bp */
+       popl    %eax            /* pt_regs->ax */
+       popl    %ds             /* pt_regs->ds */
+       popl    %es             /* pt_regs->es */
+       popl    %fs             /* pt_regs->fs */
+       popl    %gs             /* pt_regs->gs */
+       decl    %ss:early_recursion_flag
+       addl    $4, %esp        /* pop pt_regs->orig_ax */
+       iret
 
 10:
 #ifdef CONFIG_PRINTK
        xorl %eax,%eax
-       movw %ax,2(%esp)        /* clean up the segment values on some cpus */
-       movw %ax,6(%esp)
-       movw %ax,34(%esp)
+       movw %ax,PT_FS+2(%esp)  /* clean up the segment values on some cpus */
+       movw %ax,PT_DS+2(%esp)
+       movw %ax,PT_ES+2(%esp)
        leal  40(%esp),%eax
        pushl %eax              /* %esp before the exception */
        pushl %ebx
@@ -602,13 +637,6 @@ hlt_loop:
        hlt
        jmp hlt_loop
 
-ex_entry:
-       pop %es
-       pop %ds
-       pop %edx
-       pop %ecx
-       pop %eax
-       decl %ss:early_recursion_flag
 .Lis_nmi:
        addl $8,%esp            /* drop vector number and error code */
        iret
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index ffdc0e860390..f8d6dad41e8d 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -20,6 +20,7 @@
 #include <asm/processor-flags.h>
 #include <asm/percpu.h>
 #include <asm/nops.h>
+#include "../entry/calling.h"
 
 #ifdef CONFIG_PARAVIRT
 #include <asm/asm-offsets.h>
@@ -360,39 +361,52 @@ early_idt_handler_common:
        jz  1f
        incl early_recursion_flag(%rip)
 
-       pushq %rax              # 64(%rsp)
-       pushq %rcx              # 56(%rsp)
-       pushq %rdx              # 48(%rsp)
-       pushq %rsi              # 40(%rsp)
-       pushq %rdi              # 32(%rsp)
-       pushq %r8               # 24(%rsp)
-       pushq %r9               # 16(%rsp)
-       pushq %r10              #  8(%rsp)
-       pushq %r11              #  0(%rsp)
-
-       cmpl $__KERNEL_CS,96(%rsp)
+       /* The vector number is currently in the pt_regs->di slot. */
+       pushq %rsi                              /* pt_regs->si */
+       movq 8(%rsp), %rsi                      /* RSI = vector number */
+       movq %rdi, 8(%rsp)                      /* pt_regs->di = RDI */
+       pushq %rdx                              /* pt_regs->dx */
+       pushq %rcx                              /* pt_regs->cx */
+       pushq %rax                              /* pt_regs->ax */
+       pushq %r8                               /* pt_regs->r8 */
+       pushq %r9                               /* pt_regs->r9 */
+       pushq %r10                              /* pt_regs->r10 */
+       pushq %r11                              /* pt_regs->r11 */
+       pushq %rbx                              /* pt_regs->bx */
+       pushq %rbp                              /* pt_regs->bp */
+       pushq %r12                              /* pt_regs->r12 */
+       pushq %r13                              /* pt_regs->r13 */
+       pushq %r14                              /* pt_regs->r14 */
+       pushq %r15                              /* pt_regs->r15 */
+
+       cmpl $__KERNEL_CS,CS(%rsp)
        jne 11f
 
-       cmpl $14,72(%rsp)       # Page fault?
+       cmpq $14,%rsi           /* Page fault? */
        jnz 10f
-       GET_CR2_INTO(%rdi)      # can clobber any volatile register if pv
+       GET_CR2_INTO(%rdi)      /* Can clobber any volatile register if pv */
        call early_make_pgtable
        andl %eax,%eax
-       jz 20f                  # All good
+       jz 20f                  /* All good */
 
 10:
-       leaq 88(%rsp),%rdi      # Pointer to %rip
+       movq %rsp,%rdi          /* RDI = pt_regs; RSI is already trapnr */
        call early_fixup_exception
        andl %eax,%eax
        jnz 20f                 # Found an exception entry
 
 11:
 #ifdef CONFIG_EARLY_PRINTK
-       GET_CR2_INTO(%r9)       # can clobber any volatile register if pv
-       movl 80(%rsp),%r8d      # error code
-       movl 72(%rsp),%esi      # vector number
-       movl 96(%rsp),%edx      # %cs
-       movq 88(%rsp),%rcx      # %rip
+       /*
+        * On paravirt kernels, GET_CR2_INTO clobbers callee-clobbered regs.
+        * We only care about RSI, so we need to save it.
+        */
+       movq %rsi,%rbx          /* Save vector number */
+       GET_CR2_INTO(%r9)
+       movq ORIG_RAX(%rsp),%r8 /* error code */
+       movq %rbx,%rsi          /* vector number */
+       movq CS(%rsp),%rdx
+       movq RIP(%rsp),%rcx
        xorl %eax,%eax
        leaq early_idt_msg(%rip),%rdi
        call early_printk
@@ -401,24 +415,16 @@ early_idt_handler_common:
        call dump_stack
 #ifdef CONFIG_KALLSYMS 
        leaq early_idt_ripmsg(%rip),%rdi
-       movq 40(%rsp),%rsi      # %rip again
+       movq RIP(%rsp),%rsi     # %rip again
        call __print_symbol
 #endif
 #endif /* EARLY_PRINTK */
 1:     hlt
        jmp 1b
 
-20:    # Exception table entry found or page table generated
-       popq %r11
-       popq %r10
-       popq %r9
-       popq %r8
-       popq %rdi
-       popq %rsi
-       popq %rdx
-       popq %rcx
-       popq %rax
+20:    /* Exception table entry found or page table generated */
        decl early_recursion_flag(%rip)
+       jmp restore_regs_and_iret
 .Lis_nmi:
        addq $16,%rsp           # drop vector number and error code
        INTERRUPT_RETURN
diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c
index 9dd7e4b7fcde..176e48de25d4 100644
--- a/arch/x86/mm/extable.c
+++ b/arch/x86/mm/extable.c
@@ -90,13 +90,13 @@ int fixup_exception(struct pt_regs *regs, int trapnr)
 }
 
 /* Restricted version used during very early boot */
-int __init early_fixup_exception(unsigned long *ip)
+int __init early_fixup_exception(struct pt_regs *regs, int trapnr)
 {
        const struct exception_table_entry *e;
        unsigned long new_ip;
        ex_handler_t handler;
 
-       e = search_exception_tables(*ip);
+       e = search_exception_tables(regs->ip);
        if (!e)
                return 0;
 
@@ -107,7 +107,7 @@ int __init early_fixup_exception(unsigned long *ip)
        if (handler != ex_handler_default)
                return 0;
 
-       *ip = new_ip;
+       regs->ip = new_ip;
        return 1;
 }
 
-- 
2.5.5

Reply via email to