Module Name: src
Committed By: maxv
Date: Sat Jan 27 07:45:58 UTC 2018
Modified Files:
src/sys/arch/i386/i386: copy.S
Log Message:
Sync with amd64, in particular, add END() markers, don't fall through
functions, narrow the copy windows, and remove suword.
To generate a diff of this commit:
cvs rdiff -u -r1.25 -r1.26 src/sys/arch/i386/i386/copy.S
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/i386/i386/copy.S
diff -u src/sys/arch/i386/i386/copy.S:1.25 src/sys/arch/i386/i386/copy.S:1.26
--- src/sys/arch/i386/i386/copy.S:1.25 Fri Sep 16 12:28:41 2016
+++ src/sys/arch/i386/i386/copy.S Sat Jan 27 07:45:57 2018
@@ -1,7 +1,6 @@
-/* $NetBSD: copy.S,v 1.25 2016/09/16 12:28:41 maxv Exp $ */
-/* NetBSD: locore.S,v 1.34 2005/04/01 11:59:31 yamt Exp $ */
+/* $NetBSD: copy.S,v 1.26 2018/01/27 07:45:57 maxv Exp $ */
-/*-
+/*
* Copyright (c) 1998, 2000, 2004, 2008 The NetBSD Foundation, Inc.
* All rights reserved.
*
@@ -30,7 +29,7 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-/*-
+/*
* Copyright (c) 1990 The Regents of the University of California.
* All rights reserved.
*
@@ -65,7 +64,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: copy.S,v 1.25 2016/09/16 12:28:41 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: copy.S,v 1.26 2018/01/27 07:45:57 maxv Exp $");
#include "assym.h"
@@ -75,8 +74,8 @@ __KERNEL_RCSID(0, "$NetBSD: copy.S,v 1.2
#include <machine/cputypes.h>
#define GET_CURPCB(reg) \
- movl CPUVAR(CURLWP), reg; \
- movl L_PCB(reg), reg
+ movl CPUVAR(CURLWP),reg; \
+ movl L_PCB(reg),reg
/*
* These are arranged so that the abnormal case is a forwards
@@ -86,10 +85,10 @@ __KERNEL_RCSID(0, "$NetBSD: copy.S,v 1.2
#define DEFERRED_SWITCH_CHECK \
CHECK_DEFERRED_SWITCH ; \
jnz 99f ; \
- 98:
+98:
#define DEFERRED_SWITCH_CALL \
- 99: ; \
+99: ; \
call _C_LABEL(do_pmap_load) ; \
jmp 98b
@@ -109,25 +108,26 @@ LABEL(x86_copyfunc_start)
*/
NENTRY(do_pmap_load)
pushl %ebp
- movl %esp, %ebp
+ movl %esp,%ebp
pushl %ebx
- movl CPUVAR(CURLWP), %ebx
+ movl CPUVAR(CURLWP),%ebx
1:
incl L_NOPREEMPT(%ebx)
call _C_LABEL(pmap_load)
decl L_NOPREEMPT(%ebx)
jnz 2f
- cmpl $0, L_DOPREEMPT(%ebx)
+ cmpl $0,L_DOPREEMPT(%ebx)
jz 2f
pushl $0
call _C_LABEL(kpreempt)
- addl $4, %esp
+ addl $4,%esp
2:
- cmpl $0, CPUVAR(WANT_PMAPLOAD)
+ cmpl $0,CPUVAR(WANT_PMAPLOAD)
jnz 1b
popl %ebx
leave
ret
+END(do_pmap_load)
/*
* void *return_address(unsigned int level);
@@ -151,12 +151,12 @@ ENTRY(return_address)
movl 0x4(%eax),%eax
movl $0,PCB_ONFAULT(%edx)
ret
+END(return_address)
/*
* int kcopy(const void *from, void *to, size_t len);
- * Copy len bytes, abort on fault.
+ * Copy len bytes from and to kernel memory, and abort on fault.
*/
-/* LINTSTUB: Func: int kcopy(const void *from, void *to, size_t len) */
ENTRY(kcopy)
pushl %esi
pushl %edi
@@ -206,6 +206,7 @@ ENTRY(kcopy)
popl %esi
xorl %eax,%eax
ret
+END(kcopy)
/*****************************************************************************/
@@ -219,110 +220,107 @@ ENTRY(kcopy)
* Copy len bytes into the user's address space.
* see copyout(9)
*/
-
-/* LINTSTUB: Func: int copyout(const void *kaddr, void *uaddr, size_t len) */
ENTRY(copyout)
DEFERRED_SWITCH_CHECK
pushl %esi
pushl %edi
- movl 12(%esp),%esi
- movl 16(%esp),%edi
- movl 20(%esp),%eax
-.Lcopyout_start:
- /*
- * We check that the end of the destination buffer is not past the end
- * of the user's address space.
- */
+ movl 12(%esp),%esi /* from */
+ movl 16(%esp),%edi /* to */
+ movl 20(%esp),%eax /* len */
+
movl %edi,%edx
addl %eax,%edx
jc _C_LABEL(copy_efault)
cmpl $VM_MAXUSER_ADDRESS,%edx
ja _C_LABEL(copy_efault)
+
+.Lcopyout_start:
movl %eax,%ecx
shrl $2,%ecx
rep
movsl
andl $3,%eax
- jz 1f
+ jz .Lcopyout_end
movl %eax,%ecx
rep
movsb
-1:
.Lcopyout_end:
+
popl %edi
popl %esi
xorl %eax,%eax
ret
DEFERRED_SWITCH_CALL
+END(copyout)
/*
* int copyin(const void *from, void *to, size_t len);
* Copy len bytes from the user's address space.
* see copyin(9)
*/
-
-/* LINTSTUB: Func: int copyin(const void *uaddr, void *kaddr, size_t len) */
ENTRY(copyin)
DEFERRED_SWITCH_CHECK
pushl %esi
pushl %edi
- movl 12(%esp),%esi
- movl 16(%esp),%edi
- movl 20(%esp),%eax
- /*
- * We check that the end of the source buffer is not past the end of
- * the user's address space. If it's not, then we only need to check
- * that each page is readable, and the CPU will do that for us.
- */
-.Lcopyin_start:
+ movl 12(%esp),%esi /* from */
+ movl 16(%esp),%edi /* to */
+ movl 20(%esp),%eax /* len */
+
movl %esi,%edx
addl %eax,%edx
jc _C_LABEL(copy_efault)
cmpl $VM_MAXUSER_ADDRESS,%edx
ja _C_LABEL(copy_efault)
+
+.Lcopyin_start:
movl %eax,%ecx
shrl $2,%ecx
rep
movsl
andl $3,%eax
- jz 1f
+ jz .Lcopyin_end
movl %eax,%ecx
rep
movsb
-1:
.Lcopyin_end:
+
popl %edi
popl %esi
xorl %eax,%eax
ret
DEFERRED_SWITCH_CALL
+END(copyin)
-/* LINTSTUB: Ignore */
NENTRY(copy_efault)
movl $EFAULT,%eax
+ popl %edi
+ popl %esi
+ ret
+END(copy_efault)
/*
* kcopy_fault is used by kcopy and copy_fault is used by copyin/out.
*
* they're distinguished for lazy pmap switching. see trap().
*/
-/* LINTSTUB: Ignore */
+
NENTRY(kcopy_fault)
popl %edi
popl %esi
ret
+END(kcopy_fault)
-/* LINTSTUB: Ignore */
NENTRY(copy_fault)
popl %edi
popl %esi
ret
+END(copy_fault)
-/* LINTSTUB: Ignore */
NENTRY(return_address_fault)
- movl $0,PCB_ONFAULT(%edx)
- movl $0,%eax
+ movl $0,PCB_ONFAULT(%edx)
+ movl $0,%eax
ret
+END(return_address_fault)
/*
* int copyoutstr(const void *from, void *to, size_t maxlen, size_t *lencopied);
@@ -332,16 +330,14 @@ NENTRY(return_address_fault)
* return 0 or EFAULT.
* see copyoutstr(9)
*/
-/* LINTSTUB: Func: int copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) */
ENTRY(copyoutstr)
DEFERRED_SWITCH_CHECK
pushl %esi
pushl %edi
- movl 12(%esp),%esi /* esi = from */
- movl 16(%esp),%edi /* edi = to */
- movl 20(%esp),%edx /* edx = maxlen */
-.Lcopyoutstr_start:
-5:
+ movl 12(%esp),%esi /* esi = from */
+ movl 16(%esp),%edi /* edi = to */
+ movl 20(%esp),%edx /* edx = maxlen */
+
/*
* Get min(%edx, VM_MAXUSER_ADDRESS-%edi).
*/
@@ -352,16 +348,16 @@ ENTRY(copyoutstr)
jae 1f
movl %eax,%edx
movl %eax,20(%esp)
-
1: incl %edx
+.Lcopyoutstr_start:
1: decl %edx
jz 2f
lodsb
stosb
testb %al,%al
-.Lcopyoutstr_end:
jnz 1b
+.Lcopyoutstr_end:
/* Success -- 0 byte reached. */
decl %edx
@@ -374,6 +370,7 @@ ENTRY(copyoutstr)
movl $ENAMETOOLONG,%eax
jmp copystr_return
DEFERRED_SWITCH_CALL
+END(copyoutstr)
/*
* int copyinstr(const void *from, void *to, size_t maxlen, size_t *lencopied);
@@ -383,7 +380,6 @@ ENTRY(copyoutstr)
* return 0 or EFAULT.
* see copyinstr(9)
*/
-/* LINTSTUB: Func: int copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) */
ENTRY(copyinstr)
DEFERRED_SWITCH_CHECK
pushl %esi
@@ -395,7 +391,6 @@ ENTRY(copyinstr)
/*
* Get min(%edx, VM_MAXUSER_ADDRESS-%esi).
*/
-.Lcopyinstr_start:
movl $VM_MAXUSER_ADDRESS,%eax
subl %esi,%eax
jc _C_LABEL(copystr_efault)
@@ -403,16 +398,16 @@ ENTRY(copyinstr)
jae 1f
movl %eax,%edx
movl %eax,20(%esp)
-
1: incl %edx
+.Lcopyinstr_start:
1: decl %edx
jz 2f
lodsb
stosb
testb %al,%al
-.Lcopyinstr_end:
jnz 1b
+.Lcopyinstr_end:
/* Success -- 0 byte reached. */
decl %edx
@@ -425,12 +420,13 @@ ENTRY(copyinstr)
movl $ENAMETOOLONG,%eax
jmp copystr_return
DEFERRED_SWITCH_CALL
+END(copyinstr)
-/* LINTSTUB: Ignore */
NENTRY(copystr_efault)
movl $EFAULT,%eax
+ jmp copystr_return
+END(copystr_efault)
-/* LINTSTUB: Ignore */
NENTRY(copystr_fault)
copystr_return:
/* Set *lencopied and return %eax. */
@@ -444,6 +440,7 @@ copystr_return:
8: popl %edi
popl %esi
ret
+END(copystr_fault)
/*
* int copystr(const void *from, void *to, size_t maxlen, size_t *lencopied);
@@ -452,7 +449,6 @@ copystr_return:
* string is too long, return ENAMETOOLONG; else return 0.
* see copystr(9)
*/
-/* LINTSTUB: Func: int copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done) */
ENTRY(copystr)
pushl %esi
pushl %edi
@@ -488,13 +484,13 @@ ENTRY(copystr)
7: popl %edi
popl %esi
ret
+END(copystr)
/*
* long fuword(const void *uaddr);
* Fetch an int from the user's address space.
* see fuword(9)
*/
-/* LINTSTUB: Func: long fuword(const void *base) */
ENTRY(fuword)
DEFERRED_SWITCH_CHECK
movl 4(%esp),%edx
@@ -502,10 +498,13 @@ ENTRY(fuword)
ja _C_LABEL(fusuaddrfault)
GET_CURPCB(%ecx)
movl $_C_LABEL(fusufault),PCB_ONFAULT(%ecx)
+
movl (%edx),%eax
+
movl $0,PCB_ONFAULT(%ecx)
ret
DEFERRED_SWITCH_CALL
+END(fuword)
/*
* int fuswintr(const void *uaddr);
@@ -513,9 +512,8 @@ ENTRY(fuword)
* interrupt.
* see fuswintr(9)
*/
-/* LINTSTUB: Func: int fuswintr(const void *base) */
ENTRY(fuswintr)
- cmpl $TLBSTATE_VALID, CPUVAR(TLBSTATE)
+ cmpl $TLBSTATE_VALID,CPUVAR(TLBSTATE)
jnz _C_LABEL(fusuaddrfault)
movl 4(%esp),%edx
cmpl $VM_MAXUSER_ADDRESS-2,%edx
@@ -523,16 +521,18 @@ ENTRY(fuswintr)
movl CPUVAR(CURLWP),%ecx
movl L_PCB(%ecx),%ecx
movl $_C_LABEL(fusubail),PCB_ONFAULT(%ecx)
+
movzwl (%edx),%eax
+
movl $0,PCB_ONFAULT(%ecx)
ret
+END(fuswintr)
/*
* int fubyte(const void *uaddr);
* Fetch a byte from the user's address space.
* see fubyte(9)
*/
-/* LINTSTUB: Func: int fubyte(const void *base) */
ENTRY(fubyte)
DEFERRED_SWITCH_CHECK
movl 4(%esp),%edx
@@ -540,58 +540,41 @@ ENTRY(fubyte)
ja _C_LABEL(fusuaddrfault)
GET_CURPCB(%ecx)
movl $_C_LABEL(fusufault),PCB_ONFAULT(%ecx)
+
movzbl (%edx),%eax
+
movl $0,PCB_ONFAULT(%ecx)
ret
DEFERRED_SWITCH_CALL
+END(fubyte)
/*
* Handle faults from [fs]u*(). Clean up and return -1.
*/
-/* LINTSTUB: Ignore */
NENTRY(fusufault)
movl $0,PCB_ONFAULT(%ecx)
movl $-1,%eax
ret
+END(fusufault)
/*
* Handle faults from [fs]u*(). Clean up and return -1. This differs from
* fusufault() in that trap() will recognize it and return immediately rather
* than trying to page fault.
*/
-/* LINTSTUB: Ignore */
NENTRY(fusubail)
movl $0,PCB_ONFAULT(%ecx)
movl $-1,%eax
ret
+END(fusubail)
/*
* Handle earlier faults from [fs]u*(), due to our of range addresses.
*/
-/* LINTSTUB: Ignore */
NENTRY(fusuaddrfault)
movl $-1,%eax
ret
-
-/*
- * int suword(void *uaddr, long x);
- * Store an int in the user's address space.
- * see suword(9)
- */
-/* LINTSTUB: Func: int suword(void *base, long c) */
-ENTRY(suword)
- DEFERRED_SWITCH_CHECK
- movl 4(%esp),%edx
- cmpl $VM_MAXUSER_ADDRESS-4,%edx
- ja _C_LABEL(fusuaddrfault)
- GET_CURPCB(%ecx)
- movl $_C_LABEL(fusufault),PCB_ONFAULT(%ecx)
- movl 8(%esp),%eax
- movl %eax,(%edx)
- xorl %eax,%eax
- movl %eax,PCB_ONFAULT(%ecx)
- ret
- DEFERRED_SWITCH_CALL
+END(fusuaddrfault)
/*
* int suswintr(void *uaddr, short x);
@@ -599,9 +582,8 @@ ENTRY(suword)
* interrupt.
* see suswintr(9)
*/
-/* LINTSTUB: Func: int suswintr(void *base, short c) */
ENTRY(suswintr)
- cmpl $TLBSTATE_VALID, CPUVAR(TLBSTATE)
+ cmpl $TLBSTATE_VALID,CPUVAR(TLBSTATE)
jnz _C_LABEL(fusuaddrfault)
movl 4(%esp),%edx
cmpl $VM_MAXUSER_ADDRESS-2,%edx
@@ -610,17 +592,19 @@ ENTRY(suswintr)
movl L_PCB(%ecx),%ecx
movl $_C_LABEL(fusubail),PCB_ONFAULT(%ecx)
movl 8(%esp),%eax
+
movw %ax,(%edx)
+
xorl %eax,%eax
movl %eax,PCB_ONFAULT(%ecx)
ret
+END(suswintr)
/*
* int subyte(void *uaddr, char x);
* Store a byte in the user's address space.
* see subyte(9)
*/
-/* LINTSTUB: Func: int subyte(void *base, int c) */
ENTRY(subyte)
DEFERRED_SWITCH_CHECK
movl 4(%esp),%edx
@@ -629,11 +613,14 @@ ENTRY(subyte)
GET_CURPCB(%ecx)
movl $_C_LABEL(fusufault),PCB_ONFAULT(%ecx)
movb 8(%esp),%al
+
movb %al,(%edx)
+
xorl %eax,%eax
movl %eax,PCB_ONFAULT(%ecx)
ret
DEFERRED_SWITCH_CALL
+END(subyte)
/*
* Compare-and-swap the 32-bit integer in the user-space.
@@ -642,32 +629,38 @@ ENTRY(subyte)
*/
ENTRY(ucas_32)
DEFERRED_SWITCH_CHECK
- movl 4(%esp), %edx
- movl 8(%esp), %eax
- movl 12(%esp), %ecx
+ movl 4(%esp),%edx
+ movl 8(%esp),%eax
+ movl 12(%esp),%ecx
/* Fail if kernel-space */
- cmpl $VM_MAXUSER_ADDRESS-4, %edx
+ cmpl $VM_MAXUSER_ADDRESS-4,%edx
ja _C_LABEL(ucas_efault)
- /* Label for fault handler */
+
.Lucas32_start:
/* Perform the CAS */
lock
- cmpxchgl %ecx, (%edx)
+ cmpxchgl %ecx,(%edx)
.Lucas32_end:
+
/*
* Note: %eax is "old" value.
* Set the return values.
*/
- movl 16(%esp), %edx
- movl %eax, (%edx)
- xorl %eax, %eax
+ movl 16(%esp),%edx
+ movl %eax,(%edx)
+ xorl %eax,%eax
ret
DEFERRED_SWITCH_CALL
+END(ucas_32)
NENTRY(ucas_efault)
- mov $EFAULT, %eax
+ movl $EFAULT,%eax
+ ret
+END(ucas_efault)
+
NENTRY(ucas_fault)
ret
+END(ucas_fault)
/*
* int ucas_int(volatile int *uptr, int old, int new, int *ret);
@@ -694,12 +687,13 @@ ENTRY(x86_copyargs)
* is:
* (2 + SYS_MAXSYSARGS) * 4 = 10 * 4
*/
-.Lx86_copyargs_start:
movl %esi,%eax
addl $(10 * 4),%eax
jc _C_LABEL(x86_copyargs_efault)
cmpl $VM_MAXUSER_ADDRESS,%eax
ja _C_LABEL(x86_copyargs_efault)
+
+.Lx86_copyargs_start:
/* There are a maximum of 8 args + 2 for syscall indirect */
cmp $16,%ecx
movl (%esi),%eax
@@ -710,12 +704,11 @@ ENTRY(x86_copyargs)
movl 12(%esi),%ecx
movl %eax,8(%edx)
movl %ecx,12(%edx)
+
ja 2f /* Optimise since most sycalls have <= 4 args */
-1:
- popl %esi
- xorl %eax,%eax
- ret
+ jmp .Lx86_copyargs_end
2:
+
movl 16(%esi),%eax
movl 20(%esi),%ecx
movl %eax,16(%edx)
@@ -729,17 +722,23 @@ ENTRY(x86_copyargs)
movl %eax,32(%edx)
movl %ecx,36(%edx)
.Lx86_copyargs_end:
- jmp 1b
-/* LINTSTUB: Ignore */
+ popl %esi
+ xorl %eax,%eax
+ ret
+ DEFERRED_SWITCH_CALL
+END(x86_copyargs)
+
NENTRY(x86_copyargs_efault)
movl $EFAULT,%eax
+ popl %esi
+ ret
+END(x86_copyargs_efault)
-/* LINTSTUB: Ignore */
NENTRY(x86_copyargs_fault)
popl %esi
ret
- DEFERRED_SWITCH_CALL
+END(x86_copyargs_fault)
/*
* Label must be after all copy functions.
@@ -751,6 +750,7 @@ LABEL(x86_copyfunc_end)
*/
.section ".rodata"
.globl _C_LABEL(onfault_table)
+
_C_LABEL(onfault_table):
.long .Lcopyin_start
.long .Lcopyin_end
