Module Name: src
Committed By: ryo
Date: Sat Feb 29 21:34:37 UTC 2020
Modified Files:
src/sys/arch/aarch64/aarch64: fault.c pmap.c pmapboot.c
src/sys/arch/aarch64/include: pte.h
Log Message:
Fix pmap to work correctly with tagged addresses
- when fault, untag from address before passing to uvm/pmap functions
- pmap_extract() checks more strictly and consider the address tag
To generate a diff of this commit:
cvs rdiff -u -r1.11 -r1.12 src/sys/arch/aarch64/aarch64/fault.c
cvs rdiff -u -r1.65 -r1.66 src/sys/arch/aarch64/aarch64/pmap.c
cvs rdiff -u -r1.5 -r1.6 src/sys/arch/aarch64/aarch64/pmapboot.c
cvs rdiff -u -r1.11 -r1.12 src/sys/arch/aarch64/include/pte.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/aarch64/aarch64/fault.c
diff -u src/sys/arch/aarch64/aarch64/fault.c:1.11 src/sys/arch/aarch64/aarch64/fault.c:1.12
--- src/sys/arch/aarch64/aarch64/fault.c:1.11 Thu Jan 9 01:38:34 2020
+++ src/sys/arch/aarch64/aarch64/fault.c Sat Feb 29 21:34:37 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: fault.c,v 1.11 2020/01/09 01:38:34 ryo Exp $ */
+/* $NetBSD: fault.c,v 1.12 2020/02/29 21:34:37 ryo Exp $ */
/*
* Copyright (c) 2017 Ryo Shimizu <[email protected]>
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: fault.c,v 1.11 2020/01/09 01:38:34 ryo Exp $");
+__KERNEL_RCSID(0, "$NetBSD: fault.c,v 1.12 2020/02/29 21:34:37 ryo Exp $");
#include "opt_compat_netbsd32.h"
#include "opt_ddb.h"
@@ -157,6 +157,9 @@ data_abort_handler(struct trapframe *tf,
p = l->l_proc;
va = trunc_page((vaddr_t)tf->tf_far);
+ /* eliminate addresss tag if ECR_EL1.TBI[01] is enabled */
+ va = aarch64_untag_address(va);
+
if ((VM_MIN_KERNEL_ADDRESS <= va) && (va < VM_MAX_KERNEL_ADDRESS)) {
map = kernel_map;
UVMHIST_LOG(pmaphist, "use kernel_map %p", map, 0, 0, 0);
@@ -200,8 +203,8 @@ data_abort_handler(struct trapframe *tf,
if (user)
uvm_grow(p, va);
- UVMHIST_LOG(pmaphist, "uvm_fault success: va=%016llx",
- tf->tf_far, 0, 0, 0);
+ UVMHIST_LOG(pmaphist, "uvm_fault success: far=%016lx, va=%016llx",
+ tf->tf_far, va, 0, 0);
return;
}
Index: src/sys/arch/aarch64/aarch64/pmap.c
diff -u src/sys/arch/aarch64/aarch64/pmap.c:1.65 src/sys/arch/aarch64/aarch64/pmap.c:1.66
--- src/sys/arch/aarch64/aarch64/pmap.c:1.65 Sat Feb 29 21:10:09 2020
+++ src/sys/arch/aarch64/aarch64/pmap.c Sat Feb 29 21:34:37 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.65 2020/02/29 21:10:09 ryo Exp $ */
+/* $NetBSD: pmap.c,v 1.66 2020/02/29 21:34:37 ryo Exp $ */
/*
* Copyright (c) 2017 Ryo Shimizu <[email protected]>
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.65 2020/02/29 21:10:09 ryo Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.66 2020/02/29 21:34:37 ryo Exp $");
#include "opt_arm_debug.h"
#include "opt_ddb.h"
@@ -49,6 +49,7 @@ __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.6
#include <aarch64/pte.h>
#include <aarch64/armreg.h>
#include <aarch64/cpufunc.h>
+#include <aarch64/locore.h>
#include <aarch64/machdep.h>
#ifdef DDB
#include <aarch64/db_machdep.h>
@@ -281,15 +282,25 @@ phys_to_pp(paddr_t pa)
#define IN_KSEG_ADDR(va) \
IN_RANGE((va), AARCH64_KSEG_START, AARCH64_KSEG_END)
-#define KASSERT_PM_ADDR(pm, va) \
+#ifdef DIAGNOSTIC
+#define KASSERT_PM_ADDR(pm,va) \
do { \
+ int space = aarch64_addressspace(va); \
if ((pm) == pmap_kernel()) { \
+ KASSERTMSG(space == AARCH64_ADDRSPACE_UPPER, \
+ "%s: kernel pm %p: va=%016lx" \
+ " is out of upper address space\n", \
+ __func__, (pm), (va)); \
KASSERTMSG(IN_RANGE((va), VM_MIN_KERNEL_ADDRESS, \
VM_MAX_KERNEL_ADDRESS), \
"%s: kernel pm %p: va=%016lx" \
" is not kernel address\n", \
__func__, (pm), (va)); \
} else { \
+ KASSERTMSG(space == AARCH64_ADDRSPACE_LOWER, \
+ "%s: user pm %p: va=%016lx" \
+ " is out of lower address space\n", \
+ __func__, (pm), (va)); \
KASSERTMSG(IN_RANGE((va), \
VM_MIN_ADDRESS, VM_MAX_ADDRESS), \
"%s: user pm %p: va=%016lx" \
@@ -297,6 +308,9 @@ phys_to_pp(paddr_t pa)
__func__, (pm), (va)); \
} \
} while (0 /* CONSTCOND */)
+#else /* DIAGNOSTIC */
+#define KASSERT_PM_ADDR(pm,va)
+#endif /* DIAGNOSTIC */
static const struct pmap_devmap *pmap_devmap_table;
@@ -739,25 +753,56 @@ pmap_extract(struct pmap *pm, vaddr_t va
pt_entry_t *ptep, pte;
paddr_t pa;
vsize_t blocksize = 0;
+ int space;
extern char __kernel_text[];
extern char _end[];
- if (IN_RANGE(va, (vaddr_t)__kernel_text, (vaddr_t)_end)) {
- /* fast loookup */
- pa = KERN_VTOPHYS(va);
- } else if (IN_KSEG_ADDR(va)) {
- /* fast loookup. should be used only if actually mapped? */
- pa = AARCH64_KVA_TO_PA(va);
- } else {
- ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
- if (ptep == NULL)
+ space = aarch64_addressspace(va);
+ if (pm == pmap_kernel()) {
+ if (space != AARCH64_ADDRSPACE_UPPER)
return false;
- pte = *ptep;
- if (!lxpde_valid(pte))
+
+ if (IN_RANGE(va, (vaddr_t)__kernel_text, (vaddr_t)_end)) {
+ /* kernel text/data/bss are definitely linear mapped */
+ pa = KERN_VTOPHYS(va);
+ goto mapped;
+ } else if (IN_KSEG_ADDR(va)) {
+ /*
+ * also KSEG is linear mapped, but areas that have no
+ * physical memory haven't been mapped.
+ * fast lookup by using the S1E1R/PAR_EL1 registers.
+ */
+ register_t s = daif_disable(DAIF_I|DAIF_F);
+ reg_s1e1r_write(va);
+ __asm __volatile ("isb");
+ uint64_t par = reg_par_el1_read();
+ daif_enable(s);
+
+ if (par & PAR_F)
+ return false;
+ pa = (__SHIFTOUT(par, PAR_PA) << PAR_PA_SHIFT) +
+ (va & __BITS(PAR_PA_SHIFT - 1, 0));
+ goto mapped;
+ }
+ } else {
+ if (space != AARCH64_ADDRSPACE_LOWER)
return false;
- pa = lxpde_pa(pte) + (va & (blocksize - 1));
}
+ /*
+ * other areas, it isn't able to examined using the PAR_EL1 register,
+ * because the page may be in an access fault state due to
+ * reference bit emulation.
+ */
+ ptep = _pmap_pte_lookup_bs(pm, va, &blocksize);
+ if (ptep == NULL)
+ return false;
+ pte = *ptep;
+ if (!lxpde_valid(pte))
+ return false;
+ pa = lxpde_pa(pte) + (va & (blocksize - 1));
+
+ mapped:
if (pap != NULL)
*pap = pa;
return true;
@@ -769,7 +814,8 @@ vtophys(vaddr_t va)
struct pmap *pm;
paddr_t pa;
- if (va & TTBR_SEL_VA)
+ /* even if TBI is disabled, AARCH64_ADDRTOP_TAG means KVA */
+ if ((uint64_t)va & AARCH64_ADDRTOP_TAG)
pm = pmap_kernel();
else
pm = curlwp->l_proc->p_vmspace->vm_map.pmap;
@@ -791,13 +837,6 @@ _pmap_pte_lookup_bs(struct pmap *pm, vad
vsize_t blocksize;
unsigned int idx;
- if (((pm == pmap_kernel()) && ((va & TTBR_SEL_VA) == 0)) ||
- ((pm != pmap_kernel()) && ((va & TTBR_SEL_VA) != 0))) {
- blocksize = 0;
- ptep = NULL;
- goto done;
- }
-
/*
* traverse L0 -> L1 -> L2 -> L3
*/
@@ -856,6 +895,8 @@ pmap_icache_sync_range(pmap_t pm, vaddr_
vaddr_t va;
vsize_t blocksize = 0;
+ KASSERT_PM_ADDR(pm, sva);
+
pm_lock(pm);
for (va = sva; va < eva; va = (va + blocksize) & ~(blocksize - 1)) {
@@ -2553,12 +2594,18 @@ pmap_db_pteinfo(vaddr_t va, void (*pr)(c
paddr_t pa;
unsigned int idx;
- if (va & TTBR_SEL_VA) {
+ switch (aarch64_addressspace(va)) {
+ case AARCH64_ADDRSPACE_UPPER:
user = false;
ttbr = reg_ttbr1_el1_read();
- } else {
+ break;
+ case AARCH64_ADDRSPACE_LOWER:
user = true;
ttbr = reg_ttbr0_el1_read();
+ break;
+ default:
+ pr("illegal address space\n");
+ return;
}
pa = ttbr & TTBR_BADDR;
l0 = (pd_entry_t *)AARCH64_PA_TO_KVA(pa);
Index: src/sys/arch/aarch64/aarch64/pmapboot.c
diff -u src/sys/arch/aarch64/aarch64/pmapboot.c:1.5 src/sys/arch/aarch64/aarch64/pmapboot.c:1.6
--- src/sys/arch/aarch64/aarch64/pmapboot.c:1.5 Sat Feb 29 21:09:11 2020
+++ src/sys/arch/aarch64/aarch64/pmapboot.c Sat Feb 29 21:34:37 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pmapboot.c,v 1.5 2020/02/29 21:09:11 ryo Exp $ */
+/* $NetBSD: pmapboot.c,v 1.6 2020/02/29 21:34:37 ryo Exp $ */
/*
* Copyright (c) 2018 Ryo Shimizu <[email protected]>
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmapboot.c,v 1.5 2020/02/29 21:09:11 ryo Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmapboot.c,v 1.6 2020/02/29 21:34:37 ryo Exp $");
#include "opt_arm_debug.h"
#include "opt_ddb.h"
@@ -47,7 +47,6 @@ __KERNEL_RCSID(0, "$NetBSD: pmapboot.c,v
#define OPTIMIZE_TLB_CONTIG
-
static void
pmapboot_protect_entry(pt_entry_t *pte, vm_prot_t clrprot)
{
@@ -77,17 +76,21 @@ pmapboot_protect(vaddr_t sva, vaddr_t ev
paddr_t pa;
pd_entry_t *l0, *l1, *l2, *l3;
- for (va = sva; va < eva;) {
- /*
- * 0x0000xxxxxxxxxxxx -> l0 = (ttbr0_el1 & TTBR_BADDR)
- * 0xffffxxxxxxxxxxxx -> l0 = (ttbr1_el1 & TTBR_BADDR)
- */
- if (va & TTBR_SEL_VA)
- pa = (reg_ttbr1_el1_read() & TTBR_BADDR);
- else
- pa = (reg_ttbr0_el1_read() & TTBR_BADDR);
- l0 = (pd_entry_t *)AARCH64_PA_TO_KVA(pa);
+ switch (aarch64_addressspace(sva)) {
+ case AARCH64_ADDRSPACE_LOWER:
+ /* 0x0000xxxxxxxxxxxx */
+ pa = (reg_ttbr0_el1_read() & TTBR_BADDR);
+ break;
+ case AARCH64_ADDRSPACE_UPPER:
+ /* 0xFFFFxxxxxxxxxxxx */
+ pa = (reg_ttbr1_el1_read() & TTBR_BADDR);
+ break;
+ default:
+ return -1;
+ }
+ l0 = (pd_entry_t *)AARCH64_PA_TO_KVA(pa);
+ for (va = sva; va < eva;) {
idx = l0pde_index(va);
if (!l0pde_valid(l0[idx]))
return -1;
@@ -233,19 +236,22 @@ pmapboot_enter(vaddr_t va, paddr_t pa, p
attr |= LX_BLKPAG_OS_BOOT;
- while (va <= va_end) {
- /*
- * 0x0000xxxxxxxxxxxx -> l0 = (ttbr0_el1 & TTBR_BADDR)
- * 0xffffxxxxxxxxxxxx -> l0 = (ttbr1_el1 & TTBR_BADDR)
- */
- if (va & TTBR_SEL_VA) {
- l0 = (pd_entry_t *)(reg_ttbr1_el1_read() & TTBR_BADDR);
- ttbr = 1;
- } else {
- l0 = (pd_entry_t *)(reg_ttbr0_el1_read() & TTBR_BADDR);
- ttbr = 0;
- }
+ switch (aarch64_addressspace(va)) {
+ case AARCH64_ADDRSPACE_LOWER:
+ /* 0x0000xxxxxxxxxxxx */
+ l0 = (pd_entry_t *)(reg_ttbr0_el1_read() & TTBR_BADDR);
+ ttbr = 0;
+ break;
+ case AARCH64_ADDRSPACE_UPPER:
+ /* 0xFFFFxxxxxxxxxxxx */
+ l0 = (pd_entry_t *)(reg_ttbr1_el1_read() & TTBR_BADDR);
+ ttbr = 1;
+ break;
+ default:
+ return -1;
+ }
+ while (va <= va_end) {
#ifdef OPTIMIZE_TLB_CONTIG
ll = NULL;
llidx = -1;
Index: src/sys/arch/aarch64/include/pte.h
diff -u src/sys/arch/aarch64/include/pte.h:1.11 src/sys/arch/aarch64/include/pte.h:1.12
--- src/sys/arch/aarch64/include/pte.h:1.11 Fri Jan 31 09:23:58 2020
+++ src/sys/arch/aarch64/include/pte.h Sat Feb 29 21:34:37 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pte.h,v 1.11 2020/01/31 09:23:58 maxv Exp $ */
+/* $NetBSD: pte.h,v 1.12 2020/02/29 21:34:37 ryo Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -135,8 +135,6 @@ typedef uint64_t pt_entry_t; /* L3(4k) t
#define Ln_ENTRIES (1 << Ln_ENTRIES_SHIFT)
#define Ln_TABLE_SIZE (8 << Ln_ENTRIES_SHIFT)
-#define TTBR_SEL_VA __BIT(63) /* which TTBR is selected */
-
#elif defined(__arm__)
#include <arm/pte.h>
