Module Name: src
Committed By: skrll
Date: Sat Jan 18 14:40:04 UTC 2020
Modified Files:
src/sys/arch/arm/arm32: cpu.c pmap.c
src/sys/arch/arm/include: cpu.h types.h
src/sys/arch/arm/include/arm32: param.h pmap.h pte.h
src/sys/kern: init_sysctl.c
Log Message:
Use 4K pages on ARM_MMU_EXTENDED platforms (all armv[67] except RPI) by
creating a new pool l1ttpl for the userland L1 translation table which
needs to be 8KB and 8KB aligned.
Limit the pool to maxproc and add hooks to allow the sysctl changing of
maxproc to adjust the pool.
This comes at a 5% performance penalty for build.sh -j8 kernel on a
Tegra TK1.
To generate a diff of this commit:
cvs rdiff -u -r1.140 -r1.141 src/sys/arch/arm/arm32/cpu.c
cvs rdiff -u -r1.379 -r1.380 src/sys/arch/arm/arm32/pmap.c
cvs rdiff -u -r1.104 -r1.105 src/sys/arch/arm/include/cpu.h
cvs rdiff -u -r1.39 -r1.40 src/sys/arch/arm/include/types.h
cvs rdiff -u -r1.27 -r1.28 src/sys/arch/arm/include/arm32/param.h
cvs rdiff -u -r1.158 -r1.159 src/sys/arch/arm/include/arm32/pmap.h
cvs rdiff -u -r1.20 -r1.21 src/sys/arch/arm/include/arm32/pte.h
cvs rdiff -u -r1.223 -r1.224 src/sys/kern/init_sysctl.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/arm/arm32/cpu.c
diff -u src/sys/arch/arm/arm32/cpu.c:1.140 src/sys/arch/arm/arm32/cpu.c:1.141
--- src/sys/arch/arm/arm32/cpu.c:1.140 Wed Jan 15 08:34:04 2020
+++ src/sys/arch/arm/arm32/cpu.c Sat Jan 18 14:40:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.140 2020/01/15 08:34:04 mrg Exp $ */
+/* $NetBSD: cpu.c,v 1.141 2020/01/18 14:40:04 skrll Exp $ */
/*
* Copyright (c) 1995 Mark Brinicombe.
@@ -46,7 +46,7 @@
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.140 2020/01/15 08:34:04 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.141 2020/01/18 14:40:04 skrll Exp $");
#include <sys/param.h>
#include <sys/conf.h>
@@ -886,3 +886,12 @@ identify_features(device_t dv)
"pfr: [0]=%#x [1]=%#x\n",
cpu_processor_features[0], cpu_processor_features[1]);
}
+
+#ifdef _ARM_ARCH_6
+int
+cpu_maxproc_hook(int nmaxproc)
+{
+
+ return pmap_maxproc_set(nmaxproc);
+}
+#endif
Index: src/sys/arch/arm/arm32/pmap.c
diff -u src/sys/arch/arm/arm32/pmap.c:1.379 src/sys/arch/arm/arm32/pmap.c:1.380
--- src/sys/arch/arm/arm32/pmap.c:1.379 Sat Jan 18 07:52:33 2020
+++ src/sys/arch/arm/arm32/pmap.c Sat Jan 18 14:40:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.379 2020/01/18 07:52:33 skrll Exp $ */
+/* $NetBSD: pmap.c,v 1.380 2020/01/18 14:40:04 skrll Exp $ */
/*
* Copyright 2003 Wasabi Systems, Inc.
@@ -221,7 +221,7 @@
#include <arm/db_machdep.h>
#endif
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.379 2020/01/18 07:52:33 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.380 2020/01/18 14:40:04 skrll Exp $");
//#define PMAP_DEBUG
#ifdef PMAP_DEBUG
@@ -745,6 +745,7 @@ static void pmap_use_l1(pmap_t);
static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vaddr_t);
static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vaddr_t);
+static int pmap_l1tt_ctor(void *, void *, int);
static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int);
static int pmap_l2ptp_ctor(void *, void *, int);
static int pmap_l2dtable_ctor(void *, void *, int);
@@ -778,6 +779,20 @@ static void pmap_init_l1(struct l1_ttab
#endif
static vaddr_t kernel_pt_lookup(paddr_t);
+#ifdef ARM_MMU_EXTENDED
+
+static struct pool_cache pmap_l1tt_cache;
+
+static void *pmap_l1tt_alloc(struct pool *, int);
+static void pmap_l1tt_free(struct pool *, void *);
+
+static struct pool_allocator pmap_l1tt_allocator = {
+ .pa_alloc = pmap_l1tt_alloc,
+ .pa_free = pmap_l1tt_free,
+ .pa_pagesz = L1TT_SIZE,
+};
+
+#endif
/*
* Misc variables
@@ -1290,6 +1305,29 @@ pmap_modify_pv(struct vm_page_md *md, pa
return (oflags);
}
+
+#if defined(ARM_MMU_EXTENDED)
+int
+pmap_maxproc_set(int nmaxproc)
+{
+ static const char pmap_l1ttpool_warnmsg[] =
+ "WARNING: l1ttpool limit reached; increase kern.maxproc";
+
+// pool_cache_setlowat(&pmap_l1tt_cache, nmaxproc);
+
+ /*
+ * Set the hard limit on the pmap_l1tt_cache to the number
+ * of processes the kernel is to support. Log the limit
+ * reached message max once a minute.
+ */
+ pool_cache_sethardlimit(&pmap_l1tt_cache, nmaxproc,
+ pmap_l1ttpool_warnmsg, 60);
+
+ return 0;
+}
+
+#endif
+
/*
* Allocate an L1 translation table for the specified pmap.
* This is called at pmap creation time.
@@ -1298,33 +1336,11 @@ static void
pmap_alloc_l1(pmap_t pm)
{
#ifdef ARM_MMU_EXTENDED
-#ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
- struct vm_page *pg;
- bool ok __diagused;
- for (;;) {
-#ifdef PMAP_NEED_ALLOC_POOLPAGE
- pg = arm_pmap_alloc_poolpage(UVM_PGA_ZERO);
-#else
- pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
-#endif
- if (pg != NULL)
- break;
- uvm_wait("pmapl1alloc");
- }
- pm->pm_l1_pa = VM_PAGE_TO_PHYS(pg);
- vaddr_t va = pmap_direct_mapped_phys(pm->pm_l1_pa, &ok, 0);
- KASSERT(ok);
- KASSERT(va >= KERNEL_BASE);
+ vaddr_t va = (vaddr_t)pool_cache_get_paddr(&pmap_l1tt_cache, PR_WAITOK,
+ &pm->pm_l1_pa);
-#else
- KASSERTMSG(kernel_map != NULL, "pm %p", pm);
- vaddr_t va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
- UVM_KMF_WIRED|UVM_KMF_ZERO);
- KASSERT(va);
- pmap_extract(pmap_kernel(), va, &pm->pm_l1_pa);
-#endif
pm->pm_l1 = (pd_entry_t *)va;
- PTE_SYNC_RANGE(pm->pm_l1, PAGE_SIZE / sizeof(pt_entry_t));
+ PTE_SYNC_RANGE(pm->pm_l1, L1TT_SIZE / sizeof(pt_entry_t));
#else
struct l1_ttable *l1;
uint8_t domain;
@@ -1369,12 +1385,8 @@ static void
pmap_free_l1(pmap_t pm)
{
#ifdef ARM_MMU_EXTENDED
-#ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
- struct vm_page *pg = PHYS_TO_VM_PAGE(pm->pm_l1_pa);
- uvm_pagefree(pg);
-#else
- uvm_km_free(kernel_map, (vaddr_t)pm->pm_l1, PAGE_SIZE, UVM_KMF_WIRED);
-#endif
+ pool_cache_put_paddr(&pmap_l1tt_cache, (void *)pm->pm_l1, pm->pm_l1_pa);
+
pm->pm_l1 = NULL;
pm->pm_l1_pa = 0;
#else
@@ -1671,6 +1683,24 @@ pmap_free_l2_bucket(pmap_t pm, struct l2
pmap_free_l2_dtable(l2);
}
+#if defined(ARM_MMU_EXTENDED)
+/*
+ * Pool cache constructors for L1 translation tables
+ */
+
+static int
+pmap_l1tt_ctor(void *arg, void *v, int flags)
+{
+#ifndef PMAP_INCLUDE_PTE_SYNC
+#error not supported
+#endif
+
+ memset(v, 0, L1TT_SIZE);
+ PTE_SYNC_RANGE(v, L1TT_SIZE / sizeof(pt_entry_t));
+ return 0;
+}
+#endif
+
/*
* Pool cache constructors for L2 descriptor tables, metadata and pmap
* structures.
@@ -6484,6 +6514,17 @@ pmap_init(void)
pool_setlowat(&pmap_pv_pool, (PAGE_SIZE / sizeof(struct pv_entry)) * 2);
#ifdef ARM_MMU_EXTENDED
+ /*
+ * Initialise the L1 pool and cache.
+ */
+
+ pool_cache_bootstrap(&pmap_l1tt_cache, L1TT_SIZE, L1TT_SIZE,
+ 0, 0, "l1ttpl", &pmap_l1tt_allocator, IPL_NONE, pmap_l1tt_ctor,
+ NULL, NULL);
+
+ int error __diagused = pmap_maxproc_set(maxproc);
+ KASSERT(error == 0);
+
pmap_tlb_info_evcnt_attach(&pmap_tlb0_info);
#endif
@@ -6535,6 +6576,75 @@ pmap_bootstrap_pv_page_free(struct pool
}
}
+
+#if defined(ARM_MMU_EXTENDED)
+static void *
+pmap_l1tt_alloc(struct pool *pp, int flags)
+{
+ struct pglist plist;
+ vaddr_t va;
+
+ const int waitok = flags & PR_WAITOK;
+
+ int error = uvm_pglistalloc(L1TT_SIZE, 0, -1, L1TT_SIZE, 0, &plist, 1,
+ waitok);
+ if (error)
+ panic("Cannot allocate L1TT physical pages, %d", error);
+
+ struct vm_page *pg = TAILQ_FIRST(&plist);
+#if !defined( __HAVE_MM_MD_DIRECT_MAPPED_PHYS)
+
+ /* Allocate a L1 translation table VA */
+ va = uvm_km_alloc(kernel_map, L1TT_SIZE, L1TT_SIZE, UVM_KMF_VAONLY);
+ if (va == 0)
+ panic("Cannot allocate L1TT KVA");
+
+ const vaddr_t eva = va + L1TT_SIZE;
+ vaddr_t mva = va;
+ while (pg && mva < eva) {
+ paddr_t pa = VM_PAGE_TO_PHYS(pg);
+
+ pmap_kenter_pa(mva, pa,
+ VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE|PMAP_PTE);
+
+ mva += PAGE_SIZE;
+ pg = TAILQ_NEXT(pg, pageq.queue);
+ }
+ KASSERTMSG(pg == NULL && mva == eva, "pg %p mva %" PRIxVADDR
+ " eva %" PRIxVADDR, pg, mva, eva);
+#else
+ bool ok;
+ paddr_t pa = VM_PAGE_TO_PHYS(pg);
+ va = pmap_direct_mapped_phys(pa, &ok, 0);
+ KASSERT(ok);
+ KASSERT(va >= KERNEL_BASE);
+#endif
+
+ return (void *)va;
+}
+
+static void
+pmap_l1tt_free(struct pool *pp, void *v)
+{
+ vaddr_t va = (vaddr_t)v;
+
+#if !defined( __HAVE_MM_MD_DIRECT_MAPPED_PHYS)
+ uvm_km_free(kernel_map, va, L1TT_SIZE, 0);
+#else
+ paddr_t pa;
+
+ bool ok = pmap_extract(pmap_kernel(), va, &pa);
+ KASSERT(ok);
+ const paddr_t epa = pa + L1TT_SIZE;
+
+ for (; pa < epa; pa += PAGE_SIZE) {
+ struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
+ uvm_pagefree(pg);
+ }
+#endif
+}
+#endif
+
/*
* pmap_postinit()
*
Index: src/sys/arch/arm/include/cpu.h
diff -u src/sys/arch/arm/include/cpu.h:1.104 src/sys/arch/arm/include/cpu.h:1.105
--- src/sys/arch/arm/include/cpu.h:1.104 Wed Jan 15 08:34:04 2020
+++ src/sys/arch/arm/include/cpu.h Sat Jan 18 14:40:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.104 2020/01/15 08:34:04 mrg Exp $ */
+/* $NetBSD: cpu.h,v 1.105 2020/01/18 14:40:04 skrll Exp $ */
/*
* Copyright (c) 1994-1996 Mark Brinicombe.
@@ -329,6 +329,10 @@ vaddr_t cpu_uarea_alloc_idlelwp(struct c
*/
void cpu_attach(device_t, cpuid_t);
+#ifdef _ARM_ARCH_6
+int cpu_maxproc_hook(int);
+#endif
+
#endif /* !_LOCORE */
#endif /* _KERNEL */
Index: src/sys/arch/arm/include/types.h
diff -u src/sys/arch/arm/include/types.h:1.39 src/sys/arch/arm/include/types.h:1.40
--- src/sys/arch/arm/include/types.h:1.39 Fri Jan 17 20:28:14 2020
+++ src/sys/arch/arm/include/types.h Sat Jan 18 14:40:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: types.h,v 1.39 2020/01/17 20:28:14 skrll Exp $ */
+/* $NetBSD: types.h,v 1.40 2020/01/18 14:40:04 skrll Exp $ */
/*
* Copyright (c) 1990 The Regents of the University of California.
@@ -91,6 +91,7 @@ typedef int __register_t;
#define __HAVE_ATOMIC64_OPS
#endif
#if defined(_ARM_ARCH_6)
+#define __HAVE_MAXPROC_HOOK
#define __HAVE_UCAS_MP
#endif
Index: src/sys/arch/arm/include/arm32/param.h
diff -u src/sys/arch/arm/include/arm32/param.h:1.27 src/sys/arch/arm/include/arm32/param.h:1.28
--- src/sys/arch/arm/include/arm32/param.h:1.27 Wed Jun 19 09:53:39 2019
+++ src/sys/arch/arm/include/arm32/param.h Sat Jan 18 14:40:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: param.h,v 1.27 2019/06/19 09:53:39 skrll Exp $ */
+/* $NetBSD: param.h,v 1.28 2020/01/18 14:40:04 skrll Exp $ */
/*
* Copyright (c) 1994,1995 Mark Brinicombe.
@@ -46,12 +46,8 @@
* this file. */
#ifndef PGSHIFT
-#if defined(_ARM_ARCH_6)
-#define PGSHIFT 13 /* LOG2(NBPG) */
-#else
#define PGSHIFT 12 /* LOG2(NBPG) */
#endif
-#endif
#define NBPG (1 << PGSHIFT) /* bytes/page */
#define PGOFSET (NBPG - 1) /* byte offset into page */
#define NPTEPG (NBPG / sizeof(pt_entry_t)) /* PTEs per Page */
Index: src/sys/arch/arm/include/arm32/pmap.h
diff -u src/sys/arch/arm/include/arm32/pmap.h:1.158 src/sys/arch/arm/include/arm32/pmap.h:1.159
--- src/sys/arch/arm/include/arm32/pmap.h:1.158 Sun Jan 12 20:06:52 2020
+++ src/sys/arch/arm/include/arm32/pmap.h Sat Jan 18 14:40:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.158 2020/01/12 20:06:52 christos Exp $ */
+/* $NetBSD: pmap.h,v 1.159 2020/01/18 14:40:04 skrll Exp $ */
/*
* Copyright (c) 2002, 2003 Wasabi Systems, Inc.
@@ -383,6 +383,10 @@ bool pmap_extract(pmap_t, vaddr_t, paddr
void pmap_prefer(vaddr_t, vaddr_t *, int);
#endif
+#ifdef _ARM_ARCH_6
+int pmap_maxproc_set(int);
+#endif
+
void pmap_icache_sync_range(pmap_t, vaddr_t, vaddr_t);
/* Functions we use internally. */
Index: src/sys/arch/arm/include/arm32/pte.h
diff -u src/sys/arch/arm/include/arm32/pte.h:1.20 src/sys/arch/arm/include/arm32/pte.h:1.21
--- src/sys/arch/arm/include/arm32/pte.h:1.20 Wed Jun 19 09:54:15 2019
+++ src/sys/arch/arm/include/arm32/pte.h Sat Jan 18 14:40:04 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pte.h,v 1.20 2019/06/19 09:54:15 skrll Exp $ */
+/* $NetBSD: pte.h,v 1.21 2020/01/18 14:40:04 skrll Exp $ */
/*
* Copyright (c) 2001, 2002 Wasabi Systems, Inc.
@@ -141,6 +141,8 @@ typedef uint32_t pt_entry_t; /* L2 table
#define L1_TABLE_SIZE_REAL 0x4000 /* 16K */
#define L2_TABLE_SIZE_REAL 0x400 /* 1K */
+#define L1TT_SIZE 0x2000 /* 8K */
+
/*
* ARM L1 Descriptors
*/
Index: src/sys/kern/init_sysctl.c
diff -u src/sys/kern/init_sysctl.c:1.223 src/sys/kern/init_sysctl.c:1.224
--- src/sys/kern/init_sysctl.c:1.223 Thu Jan 2 15:42:27 2020
+++ src/sys/kern/init_sysctl.c Sat Jan 18 14:40:03 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: init_sysctl.c,v 1.223 2020/01/02 15:42:27 thorpej Exp $ */
+/* $NetBSD: init_sysctl.c,v 1.224 2020/01/18 14:40:03 skrll Exp $ */
/*-
* Copyright (c) 2003, 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: init_sysctl.c,v 1.223 2020/01/02 15:42:27 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: init_sysctl.c,v 1.224 2020/01/18 14:40:03 skrll Exp $");
#include "opt_sysv.h"
#include "opt_compat_netbsd.h"
@@ -874,6 +874,13 @@ sysctl_kern_maxproc(SYSCTLFN_ARGS)
if (nmaxproc > cpu_maxproc())
return (EINVAL);
#endif
+ error = 0;
+#ifdef __HAVE_MAXPROC_HOOK
+ error = cpu_maxproc_hook(nmaxproc);
+#endif
+ if (error)
+ return error;
+
maxproc = nmaxproc;
return (0);
