Module Name: src
Committed By: jdc
Date: Sun Jun 3 21:45:10 UTC 2012
Modified Files:
src/sys/arch/amd64/amd64 [netbsd-6]: trap.c vector.S
src/sys/arch/amd64/include [netbsd-6]: frameasm.h
Log Message:
Pull up revisions:
src/sys/arch/amd64/include/frameasm.h revision 1.17-1.19
src/sys/arch/amd64/amd64/vector.S revision 1.40-1.41
src/sys/arch/amd64/amd64/trap.c revision 1.71
(requested by dsl in ticket #280).
Move all the XEN differences to a single conditional.
Merge the XEN/non-XEN versions of INTRFASTEXIT and
INTR_RECURSE_HWFRAME by using extra defines.
Split INTRENTRY so that code can insert extra instructions
inside user/kernel conditional.
Add a ';' that got deleted in a slight tidyup.
Rejig the way TRAP() and ZTRAP() are defined and add Z/TRAP_NJ() that
excludes the 'jmp alltraps'.
Use the _NJ versions for trap entries with non-standard code.
Move all the KDTRACE_HOOKS code into a single block inside the
IDTVEC(trap03) code. This removes a mis-predicted from every
trap when KDTRACE_HOOKS are enabled.
Add a few blank lines, need some comments as well :-)
No functional changes intended.
Let the user of INTRENTRY_L() place a label on the 'swapgs' used
when faulting from user space.
If we get a fault setting the user %gs, or on a iret that is returning
to userspace, we must do a 'swapgs' to reload the kernel %gs_base.
Also save the %ds, %es, %fs, %gs selector values in the frame so
they can be restored if we finally return to user (probably after
an application SIGSEGV handler has fixed the error).
Without this any such fault leaves the kernel running with the wrong
%gs offset and it will most likely fault again early in trap().
Repeats until the stack tramples on something important.
iret change works, invalid %gs is a little harder to arrange.
Treat traps in kernel mode during the 'return to user' iret sequence
as user faults.
Based heavily in the i386 code with the correct opcode bytes inserted.
iret path tested, arranging for segment register errors is harder.
User %fs and %gs (32bit apps) are loaded much earlier and any errors
will generate kernel panics - there is probably code to try to stop
the invalid values being set.
To generate a diff of this commit:
cvs rdiff -u -r1.69 -r1.69.2.1 src/sys/arch/amd64/amd64/trap.c
cvs rdiff -u -r1.38.8.1 -r1.38.8.2 src/sys/arch/amd64/amd64/vector.S
cvs rdiff -u -r1.16 -r1.16.8.1 src/sys/arch/amd64/include/frameasm.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/trap.c
diff -u src/sys/arch/amd64/amd64/trap.c:1.69 src/sys/arch/amd64/amd64/trap.c:1.69.2.1
--- src/sys/arch/amd64/amd64/trap.c:1.69 Sat Feb 4 22:45:40 2012
+++ src/sys/arch/amd64/amd64/trap.c Sun Jun 3 21:45:10 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: trap.c,v 1.69 2012/02/04 22:45:40 reinoud Exp $ */
+/* $NetBSD: trap.c,v 1.69.2.1 2012/06/03 21:45:10 jdc Exp $ */
/*-
* Copyright (c) 1998, 2000 The NetBSD Foundation, Inc.
@@ -68,7 +68,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.69 2012/02/04 22:45:40 reinoud Exp $");
+__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.69.2.1 2012/06/03 21:45:10 jdc Exp $");
#include "opt_ddb.h"
#include "opt_kgdb.h"
@@ -122,6 +122,7 @@ dtrace_doubletrap_func_t dtrace_doubletr
#endif
void trap(struct trapframe *);
+void trap_return_fault_return(struct trapframe *) __dead;
const char * const trap_type[] = {
"privileged instruction fault", /* 0 T_PRIVINFLT */
@@ -200,17 +201,14 @@ trap(struct trapframe *frame)
struct lwp *l = curlwp;
struct proc *p;
struct pcb *pcb;
- extern char fusuintrfailure[], kcopy_fault[],
- resume_iret[];
+ extern char fusuintrfailure[], kcopy_fault[];
extern char IDTVEC(oosyscall)[];
extern char IDTVEC(osyscall)[];
extern char IDTVEC(syscall32)[];
-#if 0
- extern char resume_pop_ds[], resume_pop_es[];
-#endif
+#ifndef XEN
struct trapframe *vframe;
+#endif
ksiginfo_t ksi;
- void *resume;
void *onfault;
int type, error;
uint64_t cr2;
@@ -312,50 +310,76 @@ copyfault:
/*
* Check for failure during return to user mode.
+ * This can happen loading invalid values into the segment
+ * registers, or during the 'iret' itself.
*
- * XXXfvdl check for rex prefix?
- *
- * We do this by looking at the instruction we faulted on. The
- * specific instructions we recognize only happen when
+ * We do this by looking at the instruction we faulted on.
+ * The specific instructions we recognize only happen when
* returning from a trap, syscall, or interrupt.
- *
- * XXX
- * The heuristic used here will currently fail for the case of
- * one of the 2 pop instructions faulting when returning from a
- * a fast interrupt. This should not be possible. It can be
- * fixed by rearranging the trap frame so that the stack format
- * at this point is the same as on exit from a `slow'
- * interrupt.
*/
- switch (*(u_char *)frame->tf_rip) {
- case 0xcf: /* iret */
- vframe = (void *)((uint64_t)&frame->tf_rsp - 44);
- resume = resume_iret;
- break;
-/*
- * XXXfvdl these are illegal in long mode (not in compat mode, though)
- * and we do not take back the descriptors from the signal context anyway,
- * but may do so later for USER_LDT, in which case we need to intercept
- * other instructions (movl %eax, %Xs).
- */
-#if 0
- case 0x1f: /* popl %ds */
- vframe = (void *)((uint64_t)&frame->tf_rsp - 4);
- resume = resume_pop_ds;
- break;
- case 0x07: /* popl %es */
- vframe = (void *)((uint64_t)&frame->tf_rsp - 0);
- resume = resume_pop_es;
+
+kernelfault:
+#ifdef XEN
+ /*
+ * XXX: there has to be an equivalent 'problem'
+ * but I (dsl) don't know exactly what happens!
+ * For now panic the kernel.
+ */
+ goto we_re_toast;
+#else
+ KSI_INIT_TRAP(&ksi);
+ ksi.ksi_signo = SIGSEGV;
+ ksi.ksi_code = SEGV_ACCERR;
+ ksi.ksi_trap = type;
+
+ /* Get %rsp value before fault - there may be a pad word
+ * below the trap frame. */
+ vframe = (void *)frame->tf_rsp;
+ switch (*(uint16_t *)frame->tf_rip) {
+ case 0xcf48: /* iretq */
+ /*
+ * The 'iretq' instruction faulted, wo we have the
+ * 'user' registers saved after the kernel
+ * %rip:%cs:%fl:%rsp:%ss of the iret, and below that
+ * the user %rip:%cs:%fl:%rsp:%ss the 'iret' was
+ * processing.
+ * We must copy the user register back over the
+ * kernel fault frame to generate a normal stack
+ * frame (eg for sending a SIGSEGV).
+ */
+ vframe = (void *)((char *)vframe
+ - offsetof(struct trapframe, tf_rip));
+ memmove(vframe, frame,
+ offsetof(struct trapframe, tf_rip));
+ /* Set the faulting address to the user %eip */
+ ksi.ksi_addr = (void *)vframe->tf_rip;
+ break;
+ case 0x848e: /* mov 0xa8(%rsp),%es (8e 84 24 a8 00 00 00) */
+ case 0x9c8e: /* mov 0xb0(%rsp),%ds (8e 9c 24 b0 00 00 00) */
+ /*
+ * We faulted loading one if the user segment registers.
+ * The stack frame containing the user registers is
+ * still valid and pointed to by tf_rsp.
+ * Maybe we should check the iretq follows.
+ */
+ if (KERNELMODE(vframe->tf_cs, vframe->tf_eflags))
+ goto we_re_toast;
+ /* There is no valid address for the fault */
break;
-#endif
+
default:
goto we_re_toast;
}
- if (KERNELMODE(vframe->tf_cs, vframe->tf_rflags))
- goto we_re_toast;
- frame->tf_rip = (uint64_t)resume;
- return;
+ /* XXX: worry about on-stack trampolines for nested
+ * handlers?? */
+ /* Save outer frame for any signal return */
+ l->l_md.md_regs = vframe;
+ (*p->p_emul->e_trapsignal)(l, &ksi);
+ /* Return to user by reloading the user frame */
+ trap_return_fault_return(vframe);
+ /* NOTREACHED */
+#endif
case T_PROTFLT|T_USER: /* protection fault */
case T_TSSFLT|T_USER:
@@ -613,7 +637,7 @@ faultcommon:
goto copyfault;
printf("uvm_fault(%p, 0x%lx, %d) -> %x\n",
map, va, ftype, error);
- goto we_re_toast;
+ goto kernelfault;
}
if (error == ENOMEM) {
ksi.ksi_signo = SIGKILL;
Index: src/sys/arch/amd64/amd64/vector.S
diff -u src/sys/arch/amd64/amd64/vector.S:1.38.8.1 src/sys/arch/amd64/amd64/vector.S:1.38.8.2
--- src/sys/arch/amd64/amd64/vector.S:1.38.8.1 Mon Apr 23 16:34:16 2012
+++ src/sys/arch/amd64/amd64/vector.S Sun Jun 3 21:45:10 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: vector.S,v 1.38.8.1 2012/04/23 16:34:16 riz Exp $ */
+/* $NetBSD: vector.S,v 1.38.8.2 2012/06/03 21:45:10 jdc Exp $ */
/*-
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -100,40 +100,25 @@
/*****************************************************************************/
-#ifndef XEN
-#define PRE_TRAP
-#define TRAP(a) pushq $(a) ; jmp _C_LABEL(alltraps)
-#define ZTRAP(a) pushq $0 ; TRAP(a)
-#else
+#ifdef XEN
#define PRE_TRAP movq (%rsp),%rcx ; movq 8(%rsp),%r11 ; addq $0x10,%rsp
-#define POST_TRAP(a) pushq $(a) ; jmp _C_LABEL(alltraps)
-#define TRAP(a) PRE_TRAP ; POST_TRAP(a)
-#define ZTRAP(a) PRE_TRAP ; pushq $0 ; POST_TRAP(a)
+#else
+#define PRE_TRAP
#endif
-#define BPTTRAP(a) ZTRAP(a)
+#define TRAP_NJ(a) PRE_TRAP ; pushq $(a)
+#define ZTRAP_NJ(a) PRE_TRAP ; pushq $0 ; pushq $(a)
+#define TRAP(a) TRAP_NJ(a) ; jmp _C_LABEL(alltraps)
+#define ZTRAP(a) ZTRAP_NJ(a) ; jmp _C_LABEL(alltraps)
-#ifdef KDTRACE_HOOKS
- .bss
- .globl dtrace_invop_jump_addr
- .align 8
- .type dtrace_invop_jump_addr, @object
- .size dtrace_invop_jump_addr, 8
-dtrace_invop_jump_addr:
- .zero 8
- .globl dtrace_invop_calltrap_addr
- .align 8
- .type dtrace_invop_calltrap_addr, @object
- .size dtrace_invop_calltrap_addr, 8
-dtrace_invop_calltrap_addr:
- .zero 8
-#endif
.text
IDTVEC(trap00)
ZTRAP(T_DIVIDE)
+
IDTVEC(trap01)
- BPTTRAP(T_TRCTRAP)
+ ZTRAP(T_TRCTRAP)
+
IDTVEC(trap02)
#if defined(XEN)
ZTRAP(T_NMI)
@@ -167,18 +152,61 @@ IDTVEC(trap02)
addq $TF_REGSIZE+16,%rsp
iretq
#endif /* defined(XEN) */
+
IDTVEC(trap03)
- BPTTRAP(T_BPTFLT)
+#ifndef KDTRACE_HOOKS
+ ZTRAP(T_BPTFLT)
+#else
+ ZTRAP_NJ(T_BPTFLT)
+ INTRENTRY
+ STI(si)
+ /*
+ * DTrace Function Boundary Trace (fbt) probes are triggered
+ * by int3 (0xcc).
+ */
+ /* Check if there is no DTrace hook registered. */
+ cmpq $0,dtrace_invop_jump_addr
+ je calltrap
+
+ /*
+ * Set our jump address for the jump back in the event that
+ * the exception wasn't caused by DTrace at all.
+ */
+ /* XXX: This doesn't look right for SMP - unless it is a
+ * constant - so why set it everytime. (dsl) */
+ movq $calltrap, dtrace_invop_calltrap_addr(%rip)
+
+ /* Jump to the code hooked in by DTrace. */
+ movq dtrace_invop_jump_addr, %rax
+ jmpq *dtrace_invop_jump_addr
+
+ .bss
+ .globl dtrace_invop_jump_addr
+ .align 8
+ .type dtrace_invop_jump_addr, @object
+ .size dtrace_invop_jump_addr, 8
+dtrace_invop_jump_addr:
+ .zero 8
+ .globl dtrace_invop_calltrap_addr
+ .align 8
+ .type dtrace_invop_calltrap_addr, @object
+ .size dtrace_invop_calltrap_addr, 8
+dtrace_invop_calltrap_addr:
+ .zero 8
+ .text
+#endif
+
IDTVEC(trap04)
ZTRAP(T_OFLOW)
+
IDTVEC(trap05)
ZTRAP(T_BOUND)
+
IDTVEC(trap06)
ZTRAP(T_PRIVINFLT)
+
IDTVEC(trap07)
- PRE_TRAP;
- pushq $0 # dummy error code
- pushq $T_ASTFLT
+ ZTRAP_NJ(T_ASTFLT)
INTRENTRY
#ifdef DIAGNOSTIC
movl CPUVAR(ILEVEL),%ebx
@@ -186,34 +214,78 @@ IDTVEC(trap07)
movq CPUVAR(SELF),%rdi
call _C_LABEL(fpudna)
jmp .Lalltraps_checkusr
+
IDTVEC(trap08)
TRAP(T_DOUBLEFLT)
+
IDTVEC(trap09)
ZTRAP(T_FPOPFLT)
+
IDTVEC(trap0a)
TRAP(T_TSSFLT)
-IDTVEC(trap0b)
- TRAP(T_SEGNPFLT)
-IDTVEC(trap0c)
- TRAP(T_STKFLT)
-IDTVEC(trap0d)
- TRAP(T_PROTFLT)
+
+#ifdef XEN
+/*
+ * I don't believe XEN generates in-kernel traps for the
+ * equivalent of iret, if it does this code would be needed
+ * in order to copy the user segment registers into the fault frame.
+ */
+#define check_swapgs alltraps
+#endif
+
+IDTVEC(trap0b) /* #NP() Segment not present */
+ TRAP_NJ(T_SEGNPFLT)
+ jmp check_swapgs
+
+IDTVEC(trap0c) /* #SS() Stack exception */
+ TRAP_NJ(T_STKFLT)
+ jmp check_swapgs
+
+IDTVEC(trap0d) /* #GP() General protection */
+ TRAP_NJ(T_PROTFLT)
+#ifdef check_swapgs
+ jmp check_swapgs
+#else
+/* We need to worry about traps while the kernel %gs_base isn't loaded.
+ * These are either loads to %gs (only 32bit) or faults on iret during
+ * return to user. */
+check_swapgs:
+ INTRENTRY_L(3f,1:)
+2: sti
+ jmp calltrap
+3:
+ /* Trap in kernel mode. */
+ /* If faulting instruction is 'iret' we may need to do a 'swapgs'. */
+ movq TF_RIP(%rsp),%rax
+ cmpw $0xcf48,(%rax) /* Faulting instruction is iretq ? */
+ jne 5f /* Jump if not */
+ movq TF_RSP(%rsp),%rax /* Must read %rsp, may be a pad word */
+ testb $SEL_UPL,8(%rax) /* Check %cs of outer iret frame */
+ je 2b /* jump if iret was to kernel */
+ jmp 1b /* to user - must restore %gs */
+5:
+ /* Not 'iret', all moves to %gs also need a swapgs */
+ movw (%rax),%ax
+ andb $070,%ah /* mask mod/rm from mod/reg/rm */
+ cmpw $0x8e+050*256,%ax /* Any move to %gs (reg 5) */
+ jne 2b /* No - normal kernel fault */
+ jmp 1b /* Yes - restore %gs */
+#endif
+
IDTVEC(trap0e)
TRAP(T_PAGEFLT)
+
IDTVEC(intrspurious)
IDTVEC(trap0f)
- PRE_TRAP;
- pushq $0 # dummy error code
- pushq $T_ASTFLT
+ ZTRAP_NJ(T_ASTFLT)
INTRENTRY
#ifdef DIAGNOSTIC
movl CPUVAR(ILEVEL),%ebx
#endif /* DIAGNOSTIC */
jmp .Lalltraps_checkusr
+
IDTVEC(trap10)
- PRE_TRAP;
- pushq $0 # dummy error code
- pushq $T_ARITHTRAP
+ ZTRAP_NJ(T_ARITHTRAP)
.Ldo_fputrap:
INTRENTRY
#ifdef DIAGNOSTIC
@@ -227,15 +299,17 @@ IDTVEC(trap10)
1:
STI(si)
jmp calltrap
+
IDTVEC(trap11)
TRAP(T_ALIGNFLT)
+
IDTVEC(trap12)
ZTRAP(T_MCA)
+
IDTVEC(trap13)
- PRE_TRAP;
- pushq $0 # dummy error code
- pushq $T_XMM
+ ZTRAP_NJ(T_XMM)
jmp .Ldo_fputrap
+
IDTVEC(trap14)
IDTVEC(trap15)
IDTVEC(trap16)
@@ -270,25 +344,20 @@ IDTVEC(exceptions)
.quad _C_LABEL(Xtrap1e), _C_LABEL(Xtrap1f)
/*
- * If an error is detected during trap, syscall, or interrupt exit, trap() will
- * change %eip to point to one of these labels. We clean up the stack, if
- * necessary, and resume as if we were handling a general protection fault.
- * This will cause the process to get a SIGBUS.
- *
- * XXXfvdl currently unused, as pop %ds and pop %es are illegal in long
- * mode. However, if the x86-64 port is going to support USER_LDT, we
- * may need something like this after all.
+ * trap() calls here when it detects a fault in INTRFASTEXIT (loading the
+ * segment registers or during the iret itself).
+ * The address of the (possibly reconstructed) user trap frame is
+ * passed as an argument.
+ * Typically the code will have raised a SIGSEGV which will be actioned
+ * by the code below.
*/
-NENTRY(resume_iret)
- ZTRAP(T_PROTFLT)
-#if 0
-NENTRY(resume_pop_ds)
- movl $GSEL(GDATA_SEL, SEL_KPL),%eax
- movl %eax,%es
-NENTRY(resume_pop_es)
- movl $T_PROTFLT,TF_TRAPNO(%rsp)
- jmp calltrap
-#endif
+_C_LABEL(trap_return_fault_return): .globl trap_return_fault_return
+ mov %rdi,%rsp /* frame for user return */
+#ifdef DIAGNOSTIC
+ /* We can't recover the saved %rbx, so suppress warning */
+ movl CPUVAR(ILEVEL),%ebx
+#endif /* DIAGNOSTIC */
+ jmp .Lalltraps_checkusr
/*
* All traps go through here. Call the generic trap handler, and
@@ -297,30 +366,7 @@ NENTRY(resume_pop_es)
NENTRY(alltraps)
INTRENTRY
STI(si)
-#ifdef KDTRACE_HOOKS
- /*
- * DTrace Function Boundary Trace (fbt) probes are triggered
- * by int3 (0xcc) which causes the #BP (T_BPTFLT) breakpoint
- * interrupt. For all other trap types, just handle them in
- * the usual way.
- */
- cmpl $T_BPTFLT,TF_TRAPNO(%rsp)
- jne calltrap
-
- /* Check if there is no DTrace hook registered. */
- cmpq $0,dtrace_invop_jump_addr
- je calltrap
- /*
- * Set our jump address for the jump back in the event that
- * the exception wasn't caused by DTrace at all.
- */
- movq $calltrap, dtrace_invop_calltrap_addr(%rip)
-
- /* Jump to the code hooked in by DTrace. */
- movq dtrace_invop_jump_addr, %rax
- jmpq *dtrace_invop_jump_addr
-#endif
calltrap:
#ifdef DIAGNOSTIC
movl CPUVAR(ILEVEL),%ebx
@@ -405,6 +451,7 @@ IDTVEC(recurse_lapic_ipi)
pushq $T_ASTFLT
INTRENTRY
jmp 1f
+
IDTVEC(intr_lapic_ipi)
pushq $0
pushq $T_ASTFLT
@@ -451,6 +498,7 @@ IDTVEC(recurse_lapic_ltimer)
pushq $T_ASTFLT
INTRENTRY
jmp 1f
+
IDTVEC(intr_lapic_ltimer)
pushq $0
pushq $T_ASTFLT
@@ -513,8 +561,8 @@ IDTVEC(intr_ ## name ## num) ;\
pushq $0 /* dummy error code */ ;\
pushq $T_ASTFLT /* trap # for doing ASTs */ ;\
INTRENTRY ;\
- movq CPUVAR(ISOURCES) + (num) * 8, %r14 ;\
- mask(num) /* mask it in hardware */ ;\
+ movq CPUVAR(ISOURCES) + (num) * 8, %r14 ;\
+ mask(num) /* mask it in hardware */ ;\
early_ack(num) /* and allow other intrs */ ;\
testq %r14,%r14 ;\
jz 9f /* stray */ ;\
Index: src/sys/arch/amd64/include/frameasm.h
diff -u src/sys/arch/amd64/include/frameasm.h:1.16 src/sys/arch/amd64/include/frameasm.h:1.16.8.1
--- src/sys/arch/amd64/include/frameasm.h:1.16 Wed Aug 10 06:33:13 2011
+++ src/sys/arch/amd64/include/frameasm.h Sun Jun 3 21:45:10 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.16 2011/08/10 06:33:13 cherry Exp $ */
+/* $NetBSD: frameasm.h,v 1.16.8.1 2012/06/03 21:45:10 jdc Exp $ */
#ifndef _AMD64_MACHINE_FRAMEASM_H
#define _AMD64_MACHINE_FRAMEASM_H
@@ -17,7 +17,23 @@
/* Xen do not need swapgs, done by hypervisor */
#define swapgs
#define iretq pushq $0 ; jmp HYPERVISOR_iret
-#endif
+#define XEN_ONLY2(x,y) x,y
+#define NOT_XEN(x)
+
+#define CLI(temp_reg) \
+ movq CPUVAR(VCPU),%r ## temp_reg ; \
+ movb $1,EVTCHN_UPCALL_MASK(%r ## temp_reg);
+
+#define STI(temp_reg) \
+ movq CPUVAR(VCPU),%r ## temp_reg ; \
+ movb $0,EVTCHN_UPCALL_MASK(%r ## temp_reg);
+
+#else /* XEN */
+#define XEN_ONLY2(x,y)
+#define NOT_XEN(x) x
+#define CLI(temp_reg) cli
+#define STI(temp_reg) sti
+#endif /* XEN */
/*
* These are used on interrupt or trap entry or exit.
@@ -57,23 +73,28 @@
movq TF_RBX(%rsp),%rbx ; \
movq TF_RAX(%rsp),%rax
-#define INTRENTRY \
+#define INTRENTRY_L(kernel_trap, usertrap) \
subq $TF_REGSIZE,%rsp ; \
- testq $SEL_UPL,TF_CS(%rsp) ; \
- je 98f ; \
+ INTR_SAVE_GPRS ; \
+ testb $SEL_UPL,TF_CS(%rsp) ; \
+ je kernel_trap ; \
+usertrap ; \
swapgs ; \
movw %gs,TF_GS(%rsp) ; \
movw %fs,TF_FS(%rsp) ; \
movw %es,TF_ES(%rsp) ; \
- movw %ds,TF_DS(%rsp) ; \
-98: INTR_SAVE_GPRS
+ movw %ds,TF_DS(%rsp)
+
+#define INTRENTRY \
+ INTRENTRY_L(98f,) ; \
+98:
-#ifndef XEN
#define INTRFASTEXIT \
INTR_RESTORE_GPRS ; \
testq $SEL_UPL,TF_CS(%rsp) /* Interrupted %cs */ ; \
je 99f ; \
- cli ; \
+/* XEN: Disabling events before going to user mode sounds like a BAD idea */ \
+ NOT_XEN(cli;) \
movw TF_ES(%rsp),%es ; \
movw TF_DS(%rsp),%ds ; \
swapgs ; \
@@ -88,41 +109,15 @@
pushfq ; \
movl %cs,%r11d ; \
pushq %r11 ; \
+/* XEN: We must fixup CS, as even kernel mode runs at CPL 3 */ \
+ XEN_ONLY2(andb $0xfc,(%rsp);) \
pushq %r13 ;
-#else /* !XEN */
-/*
- * Disabling events before going to user mode sounds like a BAD idea
- * do no restore gs either, HYPERVISOR_iret will do a swapgs
- */
-#define INTRFASTEXIT \
- INTR_RESTORE_GPRS ; \
- testq $SEL_UPL,TF_CS(%rsp) ; \
- je 99f ; \
- movw TF_ES(%rsp),%es ; \
- movw TF_DS(%rsp),%ds ; \
-99: addq $TF_REGSIZE+16,%rsp /* + T_xxx and error code */ ; \
- iretq
-
-/* We must fixup CS, as even kernel mode runs at CPL 3 */
-#define INTR_RECURSE_HWFRAME \
- movq %rsp,%r10 ; \
- movl %ss,%r11d ; \
- pushq %r11 ; \
- pushq %r10 ; \
- pushfq ; \
- movl %cs,%r11d ; \
- pushq %r11 ; \
- andb $0xfc,(%rsp) ; \
- pushq %r13 ;
-
-#endif /* !XEN */
-
#define DO_DEFERRED_SWITCH \
cmpl $0, CPUVAR(WANT_PMAPLOAD) ; \
jz 1f ; \
call _C_LABEL(do_pmap_load) ; \
- 1:
+1:
#define CHECK_DEFERRED_SWITCH \
cmpl $0, CPUVAR(WANT_PMAPLOAD)
@@ -130,18 +125,4 @@
#define CHECK_ASTPENDING(reg) cmpl $0, L_MD_ASTPENDING(reg)
#define CLEAR_ASTPENDING(reg) movl $0, L_MD_ASTPENDING(reg)
-#ifdef XEN
-#define CLI(temp_reg) \
- movq CPUVAR(VCPU),%r ## temp_reg ; \
- movb $1,EVTCHN_UPCALL_MASK(%r ## temp_reg);
-
-#define STI(temp_reg) \
- movq CPUVAR(VCPU),%r ## temp_reg ; \
- movb $0,EVTCHN_UPCALL_MASK(%r ## temp_reg);
-
-#else /* XEN */
-#define CLI(temp_reg) cli
-#define STI(temp_reg) sti
-#endif /* XEN */
-
#endif /* _AMD64_MACHINE_FRAMEASM_H */
