Module Name: src
Committed By: ad
Date: Wed May 27 19:33:40 UTC 2020
Modified Files:
src/sys/arch/amd64/amd64: cpufunc.S locore.S
src/sys/arch/i386/i386: cpufunc.S locore.S
src/sys/arch/x86/include: pmap.h
src/sys/arch/x86/x86: pmap.c
Log Message:
- Add a couple of wrapper functions around STOS and MOVS and use them to zero
and copy PTEs in preference to memset()/memcpy().
- Remove related SSE / pageidlezero stuff.
To generate a diff of this commit:
cvs rdiff -u -r1.56 -r1.57 src/sys/arch/amd64/amd64/cpufunc.S
cvs rdiff -u -r1.208 -r1.209 src/sys/arch/amd64/amd64/locore.S
cvs rdiff -u -r1.42 -r1.43 src/sys/arch/i386/i386/cpufunc.S
cvs rdiff -u -r1.184 -r1.185 src/sys/arch/i386/i386/locore.S
cvs rdiff -u -r1.121 -r1.122 src/sys/arch/x86/include/pmap.h
cvs rdiff -u -r1.395 -r1.396 src/sys/arch/x86/x86/pmap.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/cpufunc.S
diff -u src/sys/arch/amd64/amd64/cpufunc.S:1.56 src/sys/arch/amd64/amd64/cpufunc.S:1.57
--- src/sys/arch/amd64/amd64/cpufunc.S:1.56 Wed May 20 18:52:48 2020
+++ src/sys/arch/amd64/amd64/cpufunc.S Wed May 27 19:33:40 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.S,v 1.56 2020/05/20 18:52:48 maxv Exp $ */
+/* $NetBSD: cpufunc.S,v 1.57 2020/05/27 19:33:40 ad Exp $ */
/*
* Copyright (c) 1998, 2007, 2008, 2020 The NetBSD Foundation, Inc.
@@ -420,3 +420,24 @@ ENTRY(outl)
outl %eax, %dx
ret
END(outl)
+
+ENTRY(x86_stos)
+ pushq %rbp
+ movq %rsp,%rbp
+ movq %rsi,%rax
+ movq %rdx,%rcx
+ rep
+ stosq
+ leave
+ ret
+END(x86_stos)
+
+ENTRY(x86_movs)
+ pushq %rbp
+ movq %rsp,%rbp
+ movq %rdx,%rcx
+ rep
+ movsq
+ leave
+ ret
+END(x86_stos)
Index: src/sys/arch/amd64/amd64/locore.S
diff -u src/sys/arch/amd64/amd64/locore.S:1.208 src/sys/arch/amd64/amd64/locore.S:1.209
--- src/sys/arch/amd64/amd64/locore.S:1.208 Tue May 5 06:32:43 2020
+++ src/sys/arch/amd64/amd64/locore.S Wed May 27 19:33:40 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.208 2020/05/05 06:32:43 maxv Exp $ */
+/* $NetBSD: locore.S,v 1.209 2020/05/27 19:33:40 ad Exp $ */
/*
* Copyright-o-rama!
@@ -1560,61 +1560,6 @@ LABEL(syscall_sysret)
END(syscall_sysret)
TEXT_USER_END
-/*
- * bool sse2_idlezero_page(void *pg)
- *
- * Zero a page without polluting the cache. Preemption must be
- * disabled by the caller. Abort if a preemption is pending.
- * Returns true if the page is zeroed, false if not.
- */
-ENTRY(sse2_idlezero_page)
- pushq %rbp
- movq %rsp,%rbp
- movl $(PAGE_SIZE/64), %ecx
- xorq %rax, %rax
- .align 16
-1:
- cmpl $0, CPUVAR(RESCHED)
- jnz 2f
- movnti %rax, 0(%rdi)
- movnti %rax, 8(%rdi)
- movnti %rax, 16(%rdi)
- movnti %rax, 24(%rdi)
- movnti %rax, 32(%rdi)
- movnti %rax, 40(%rdi)
- movnti %rax, 48(%rdi)
- movnti %rax, 56(%rdi)
- addq $64, %rdi
- decl %ecx
- jnz 1b
- sfence
- incl %eax
- popq %rbp
- KMSAN_INIT_RET(1)
- ret
-2:
- sfence
- popq %rbp
- KMSAN_INIT_RET(1)
- ret
-END(sse2_idlezero_page)
-
-/*
- * void pagezero(vaddr_t va)
- *
- * Zero a page.
- */
-ENTRY(pagezero)
- pushq %rbp
- movq %rsp,%rbp
- movq $(PAGE_SIZE / 8),%rcx
- xorq %rax,%rax
- rep
- stosq
- leave
- ret
-END(pagezero)
-
TEXT_USER_BEGIN
/*
Index: src/sys/arch/i386/i386/cpufunc.S
diff -u src/sys/arch/i386/i386/cpufunc.S:1.42 src/sys/arch/i386/i386/cpufunc.S:1.43
--- src/sys/arch/i386/i386/cpufunc.S:1.42 Wed May 27 18:47:13 2020
+++ src/sys/arch/i386/i386/cpufunc.S Wed May 27 19:33:40 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.S,v 1.42 2020/05/27 18:47:13 ad Exp $ */
+/* $NetBSD: cpufunc.S,v 1.43 2020/05/27 19:33:40 ad Exp $ */
/*-
* Copyright (c) 1998, 2007, 2020 The NetBSD Foundation, Inc.
@@ -38,7 +38,7 @@
#include <sys/errno.h>
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: cpufunc.S,v 1.42 2020/05/27 18:47:13 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpufunc.S,v 1.43 2020/05/27 19:33:40 ad Exp $");
#include "opt_xen.h"
@@ -318,3 +318,27 @@ ENTRY(outl)
outl %eax, %dx
ret
END(outl)
+
+ENTRY(x86_stos)
+ pushl %ebp
+ movl %esp,%ebp
+ movl 8(%ebp),%edi
+ movl 12(%ebp),%eax
+ movl 16(%ebp),%ecx
+ rep
+ stosl
+ leave
+ ret
+END(x86_stos)
+
+ENTRY(x86_movs)
+ pushl %ebp
+ movl %esp,%ebp
+ movl 8(%ebp),%edi
+ movl 12(%ebp),%esi
+ movl 16(%ebp),%ecx
+ rep
+ movsl
+ leave
+ ret
+END(x86_stos)
Index: src/sys/arch/i386/i386/locore.S
diff -u src/sys/arch/i386/i386/locore.S:1.184 src/sys/arch/i386/i386/locore.S:1.185
--- src/sys/arch/i386/i386/locore.S:1.184 Tue May 5 09:07:35 2020
+++ src/sys/arch/i386/i386/locore.S Wed May 27 19:33:40 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.184 2020/05/05 09:07:35 bouyer Exp $ */
+/* $NetBSD: locore.S,v 1.185 2020/05/27 19:33:40 ad Exp $ */
/*
* Copyright-o-rama!
@@ -128,7 +128,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: locore.S,v 1.184 2020/05/05 09:07:35 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: locore.S,v 1.185 2020/05/27 19:33:40 ad Exp $");
#include "opt_copy_symtab.h"
#include "opt_ddb.h"
@@ -1641,43 +1641,6 @@ ENTRY(npx586bug1)
ret
END(npx586bug1)
-/*
- * void sse2_idlezero_page(void *pg)
- *
- * Zero a page without polluting the cache. Preemption must be
- * disabled by the caller. Abort if a preemption is pending.
- */
-ENTRY(sse2_idlezero_page)
- pushl %ebp
- movl %esp,%ebp
- movl 8(%esp), %edx
- movl $(PAGE_SIZE/32), %ecx
- xorl %eax, %eax
- .align 16
-1:
- cmpl $0, CPUVAR(RESCHED)
- jnz 2f
- movnti %eax, 0(%edx)
- movnti %eax, 4(%edx)
- movnti %eax, 8(%edx)
- movnti %eax, 12(%edx)
- movnti %eax, 16(%edx)
- movnti %eax, 20(%edx)
- movnti %eax, 24(%edx)
- movnti %eax, 28(%edx)
- addl $32, %edx
- decl %ecx
- jnz 1b
- sfence
- incl %eax
- pop %ebp
- ret
-2:
- sfence
- popl %ebp
- ret
-END(sse2_idlezero_page)
-
ENTRY(intrfastexit)
movw TF_GS(%esp),%gs
movw TF_FS(%esp),%fs
Index: src/sys/arch/x86/include/pmap.h
diff -u src/sys/arch/x86/include/pmap.h:1.121 src/sys/arch/x86/include/pmap.h:1.122
--- src/sys/arch/x86/include/pmap.h:1.121 Tue May 26 10:10:31 2020
+++ src/sys/arch/x86/include/pmap.h Wed May 27 19:33:40 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.121 2020/05/26 10:10:31 bouyer Exp $ */
+/* $NetBSD: pmap.h,v 1.122 2020/05/27 19:33:40 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -431,12 +431,6 @@ void pmap_tlb_intr(void);
#define PMAP_FORK /* turn on pmap_fork interface */
/*
- * Do idle page zero'ing uncached to avoid polluting the cache.
- */
-bool pmap_pageidlezero(paddr_t);
-#define PMAP_PAGEIDLEZERO(pa) pmap_pageidlezero((pa))
-
-/*
* inline functions
*/
@@ -546,7 +540,6 @@ kvtopte(vaddr_t va)
paddr_t vtophys(vaddr_t);
vaddr_t pmap_map(vaddr_t, paddr_t, paddr_t, vm_prot_t);
void pmap_cpu_init_late(struct cpu_info *);
-bool sse2_idlezero_page(void *);
#ifdef XENPV
#include <sys/bitops.h>
@@ -617,10 +610,11 @@ extern vaddr_t pmap_direct_end;
#define PMAP_MAP_POOLPAGE(pa) PMAP_DIRECT_MAP((pa))
#define PMAP_UNMAP_POOLPAGE(va) PMAP_DIRECT_UNMAP((va))
-void pagezero(vaddr_t);
-
#endif /* __HAVE_DIRECT_MAP */
+void x86_stos(void *, long, long);
+void x86_movs(void *, void *, long);
+
#endif /* _KERNEL */
#endif /* _X86_PMAP_H_ */
Index: src/sys/arch/x86/x86/pmap.c
diff -u src/sys/arch/x86/x86/pmap.c:1.395 src/sys/arch/x86/x86/pmap.c:1.396
--- src/sys/arch/x86/x86/pmap.c:1.395 Wed May 27 19:26:43 2020
+++ src/sys/arch/x86/x86/pmap.c Wed May 27 19:33:40 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.395 2020/05/27 19:26:43 ad Exp $ */
+/* $NetBSD: pmap.c,v 1.396 2020/05/27 19:33:40 ad Exp $ */
/*
* Copyright (c) 2008, 2010, 2016, 2017, 2019, 2020 The NetBSD Foundation, Inc.
@@ -130,7 +130,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.395 2020/05/27 19:26:43 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.396 2020/05/27 19:33:40 ad Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -1297,7 +1297,8 @@ pmap_bootstrap(vaddr_t kva_start)
xen_dummy_user_pgd = xen_dummy_page - KERNBASE;
/* Zero fill it, the less checks in Xen it requires the better */
- memset((void *)(xen_dummy_user_pgd + KERNBASE), 0, PAGE_SIZE);
+ x86_stos((void *)(xen_dummy_user_pgd + KERNBASE), 0,
+ PAGE_SIZE / sizeof(long));
/* Mark read-only */
HYPERVISOR_update_va_mapping(xen_dummy_user_pgd + KERNBASE,
pmap_pa2pte(xen_dummy_user_pgd) | PTE_P | pmap_pg_nx,
@@ -1546,7 +1547,7 @@ pmap_init_pcpu(void)
pa = pmap_bootstrap_palloc(1);
*pte = (pa & PTE_FRAME) | pteflags;
pmap_update_pg(tmpva);
- memset((void *)tmpva, 0, PAGE_SIZE);
+ x86_stos((void *)tmpva, 0, PAGE_SIZE / sizeof(long));
L4_BASE[L4e_idx+i] = pa | pteflags | PTE_A;
}
@@ -1560,7 +1561,7 @@ pmap_init_pcpu(void)
pa = pmap_bootstrap_palloc(1);
*pte = (pa & PTE_FRAME) | pteflags;
pmap_update_pg(tmpva);
- memset((void *)tmpva, 0, PAGE_SIZE);
+ x86_stos((void *)tmpva, 0, PAGE_SIZE / sizeof(long));
L3_BASE[L3e_idx+i] = pa | pteflags | PTE_A;
}
@@ -1575,7 +1576,7 @@ pmap_init_pcpu(void)
pa = pmap_bootstrap_palloc(1);
*pte = (pa & PTE_FRAME) | pteflags;
pmap_update_pg(tmpva);
- memset((void *)tmpva, 0, PAGE_SIZE);
+ x86_stos((void *)tmpva, 0, PAGE_SIZE / sizeof(long));
L2_BASE[L2e_idx+i] = pa | pteflags | PTE_A;
}
@@ -1667,7 +1668,7 @@ pmap_init_directmap(struct pmap *kpm)
pa = pmap_bootstrap_palloc(1);
*pte = (pa & PTE_FRAME) | pteflags;
pmap_update_pg(tmpva);
- memset((void *)tmpva, 0, PAGE_SIZE);
+ x86_stos((void *)tmpva, 0, PAGE_SIZE / sizeof(long));
L4_BASE[L4e_idx+i] = pa | pteflags | PTE_A;
}
@@ -1681,7 +1682,7 @@ pmap_init_directmap(struct pmap *kpm)
pa = pmap_bootstrap_palloc(1);
*pte = (pa & PTE_FRAME) | pteflags;
pmap_update_pg(tmpva);
- memset((void *)tmpva, 0, PAGE_SIZE);
+ x86_stos((void *)tmpva, 0, PAGE_SIZE / sizeof(long));
L3_BASE[L3e_idx+i] = pa | pteflags | PTE_A;
}
@@ -2631,7 +2632,7 @@ pmap_pdp_init(pd_entry_t *pdir)
int s;
#endif
- memset(pdir, 0, PDP_SIZE * PAGE_SIZE);
+ x86_stos(pdir, 0, PDP_SIZE * PAGE_SIZE / sizeof(long));
/*
* NOTE: This is all done unlocked, but we will check afterwards
@@ -2674,8 +2675,8 @@ pmap_pdp_init(pd_entry_t *pdir)
/* Copy the kernel's top level PDE */
npde = nkptp[PTP_LEVELS - 1];
- memcpy(&pdir[PDIR_SLOT_KERN], &PDP_BASE[PDIR_SLOT_KERN],
- npde * sizeof(pd_entry_t));
+ x86_movs(&pdir[PDIR_SLOT_KERN], &PDP_BASE[PDIR_SLOT_KERN],
+ npde * sizeof(pd_entry_t) / sizeof(long));
if (VM_MIN_KERNEL_ADDRESS != KERNBASE) {
int idx = pl_i(KERNBASE, PTP_LEVELS);
@@ -3810,7 +3811,7 @@ void
pmap_zero_page(paddr_t pa)
{
#if defined(__HAVE_DIRECT_MAP)
- pagezero(PMAP_DIRECT_MAP(pa));
+ x86_stos((void *)PMAP_DIRECT_MAP(pa), 0, PAGE_SIZE / sizeof(long));
#else
#if defined(XENPV)
if (XEN_VERSION_SUPPORTED(3, 4))
@@ -3834,7 +3835,7 @@ pmap_zero_page(paddr_t pa)
pmap_pte_flush();
pmap_update_pg(zerova); /* flush TLB */
- memset((void *)zerova, 0, PAGE_SIZE);
+ x86_stos((void *)zerova, 0, PAGE_SIZE / sizeof(long));
#if defined(DIAGNOSTIC) || defined(XENPV)
pmap_pte_set(zpte, 0); /* zap ! */
@@ -3845,47 +3846,6 @@ pmap_zero_page(paddr_t pa)
#endif /* defined(__HAVE_DIRECT_MAP) */
}
-/*
- * pmap_pagezeroidle: the same, for the idle loop page zero'er.
- * Returns true if the page was zero'd, false if we aborted for
- * some reason.
- */
-bool
-pmap_pageidlezero(paddr_t pa)
-{
-#ifdef __HAVE_DIRECT_MAP
- KASSERT(cpu_feature[0] & CPUID_SSE2);
- return sse2_idlezero_page((void *)PMAP_DIRECT_MAP(pa));
-#else
- struct cpu_info *ci;
- pt_entry_t *zpte;
- vaddr_t zerova;
- bool rv;
-
- const pd_entry_t pteflags = PTE_P | PTE_W | pmap_pg_nx | PTE_D | PTE_A;
-
- ci = curcpu();
- zerova = ci->vpage[VPAGE_ZER];
- zpte = ci->vpage_pte[VPAGE_ZER];
-
- KASSERT(cpu_feature[0] & CPUID_SSE2);
- KASSERT(*zpte == 0);
-
- pmap_pte_set(zpte, pmap_pa2pte(pa) | pteflags);
- pmap_pte_flush();
- pmap_update_pg(zerova); /* flush TLB */
-
- rv = sse2_idlezero_page((void *)zerova);
-
-#if defined(DIAGNOSTIC) || defined(XENPV)
- pmap_pte_set(zpte, 0); /* zap ! */
- pmap_pte_flush();
-#endif
-
- return rv;
-#endif
-}
-
void
pmap_copy_page(paddr_t srcpa, paddr_t dstpa)
{
@@ -3893,7 +3853,7 @@ pmap_copy_page(paddr_t srcpa, paddr_t ds
vaddr_t srcva = PMAP_DIRECT_MAP(srcpa);
vaddr_t dstva = PMAP_DIRECT_MAP(dstpa);
- memcpy((void *)dstva, (void *)srcva, PAGE_SIZE);
+ x86_movs((void *)dstva, (void *)srcva, PAGE_SIZE / sizeof(long));
#else
#if defined(XENPV)
if (XEN_VERSION_SUPPORTED(3, 4)) {
@@ -3923,7 +3883,7 @@ pmap_copy_page(paddr_t srcpa, paddr_t ds
pmap_update_pg(srcva);
pmap_update_pg(dstva);
- memcpy((void *)dstva, (void *)srcva, PAGE_SIZE);
+ x86_movs((void *)dstva, (void *)srcva, PAGE_SIZE / sizeof(long));
#if defined(DIAGNOSTIC) || defined(XENPV)
pmap_pte_set(srcpte, 0);
@@ -5447,7 +5407,8 @@ pmap_get_physpage(void)
if (!uvm_page_physget(&pa))
panic("%s: out of memory", __func__);
#if defined(__HAVE_DIRECT_MAP)
- pagezero(PMAP_DIRECT_MAP(pa));
+ x86_stos((void *)PMAP_DIRECT_MAP(pa), 0,
+ PAGE_SIZE / sizeof(long));
#else
#if defined(XENPV)
if (XEN_VERSION_SUPPORTED(3, 4)) {
@@ -5460,7 +5421,7 @@ pmap_get_physpage(void)
PTE_W | pmap_pg_nx);
pmap_pte_flush();
pmap_update_pg((vaddr_t)early_zerop);
- memset(early_zerop, 0, PAGE_SIZE);
+ x86_stos(early_zerop, 0, PAGE_SIZE / sizeof(long));
#if defined(DIAGNOSTIC) || defined(XENPV)
pmap_pte_set(early_zero_pte, 0);
pmap_pte_flush();
@@ -5658,9 +5619,9 @@ pmap_growkernel(vaddr_t maxkvaddr)
*/
continue;
}
- memcpy(&pm->pm_pdir[PDIR_SLOT_KERN + old],
+ x86_movs(&pm->pm_pdir[PDIR_SLOT_KERN + old],
&kpm->pm_pdir[PDIR_SLOT_KERN + old],
- newpdes * sizeof(pd_entry_t));
+ newpdes * sizeof(pd_entry_t) / sizeof(long));
}
mutex_exit(&pmaps_lock);
#endif
@@ -5828,13 +5789,13 @@ pmap_init_tmp_pgtbl(paddr_t pg)
/* Zero levels 1-3 */
for (level = 0; level < PTP_LEVELS - 1; ++level) {
tmp_pml = (void *)x86_tmp_pml_vaddr[level];
- memset(tmp_pml, 0, PAGE_SIZE);
+ x86_stos(tmp_pml, 0, PAGE_SIZE / sizeof(long));
}
/* Copy PML4 */
kernel_pml = pmap_kernel()->pm_pdir;
tmp_pml = (void *)x86_tmp_pml_vaddr[PTP_LEVELS - 1];
- memcpy(tmp_pml, kernel_pml, PAGE_SIZE);
+ x86_movs(tmp_pml, kernel_pml, PAGE_SIZE / sizeof(long));
#ifdef PAE
/*
@@ -6747,7 +6708,7 @@ pmap_ept_transform(struct pmap *pmap)
pmap->pm_write_protect = pmap_ept_write_protect;
pmap->pm_unwire = pmap_ept_unwire;
- memset(pmap->pm_pdir, 0, PAGE_SIZE);
+ x86_stos(pmap->pm_pdir, 0, PAGE_SIZE / sizeof(long));
}
#endif /* __HAVE_DIRECT_MAP && __x86_64__ && !XENPV */
