Module Name: src
Committed By: riastradh
Date: Mon May 11 21:37:31 UTC 2020
Modified Files:
src/sys/kern: subr_percpu.c
Log Message:
Teach percpu to run constructors on CPU attach.
This should make percpu work reliably for things like cprng_strong
much earlier -- not just after all CPUs have been detected. This is
also necessary (though not on its own sufficient) for CPU hotplug.
To generate a diff of this commit:
cvs rdiff -u -r1.24 -r1.25 src/sys/kern/subr_percpu.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/subr_percpu.c
diff -u src/sys/kern/subr_percpu.c:1.24 src/sys/kern/subr_percpu.c:1.25
--- src/sys/kern/subr_percpu.c:1.24 Fri Feb 7 11:55:22 2020
+++ src/sys/kern/subr_percpu.c Mon May 11 21:37:31 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: subr_percpu.c,v 1.24 2020/02/07 11:55:22 thorpej Exp $ */
+/* $NetBSD: subr_percpu.c,v 1.25 2020/05/11 21:37:31 riastradh Exp $ */
/*-
* Copyright (c)2007,2008 YAMAMOTO Takashi,
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_percpu.c,v 1.24 2020/02/07 11:55:22 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_percpu.c,v 1.25 2020/05/11 21:37:31 riastradh Exp $");
#include <sys/param.h>
#include <sys/cpu.h>
@@ -50,8 +50,10 @@ __KERNEL_RCSID(0, "$NetBSD: subr_percpu.
struct percpu {
unsigned pc_offset;
size_t pc_size;
+ percpu_callback_t pc_ctor;
percpu_callback_t pc_dtor;
void *pc_cookie;
+ LIST_ENTRY(percpu) pc_list;
};
static krwlock_t percpu_swap_lock __cacheline_aligned;
@@ -59,6 +61,9 @@ static vmem_t * percpu_offset_arena __r
static struct {
kmutex_t lock;
unsigned int nextoff;
+ LIST_HEAD(, percpu) ctor_list;
+ struct lwp *busy;
+ kcondvar_t cv;
} percpu_allocation __cacheline_aligned;
static percpu_cpu_t *
@@ -217,6 +222,9 @@ percpu_init(void)
rw_init(&percpu_swap_lock);
mutex_init(&percpu_allocation.lock, MUTEX_DEFAULT, IPL_NONE);
percpu_allocation.nextoff = PERCPU_QUANTUM_SIZE;
+ LIST_INIT(&percpu_allocation.ctor_list);
+ percpu_allocation.busy = NULL;
+ cv_init(&percpu_allocation.cv, "percpu");
percpu_offset_arena = vmem_xcreate("percpu", 0, 0, PERCPU_QUANTUM_SIZE,
percpu_backend_alloc, NULL, NULL, PERCPU_QCACHE_MAX, VM_SLEEP,
@@ -227,18 +235,50 @@ percpu_init(void)
* percpu_init_cpu: cpu initialization
*
* => should be called before the cpu appears on the list for CPU_INFO_FOREACH.
+ * => may be called for static CPUs afterward (typically just primary CPU)
*/
void
percpu_init_cpu(struct cpu_info *ci)
{
percpu_cpu_t * const pcc = cpu_percpu(ci);
+ struct percpu *pc;
size_t size = percpu_allocation.nextoff; /* XXX racy */
ASSERT_SLEEPABLE();
+
+ /*
+ * For the primary CPU, prior percpu_create may have already
+ * triggered allocation, so there's nothing more for us to do
+ * here.
+ */
+ if (pcc->pcc_size)
+ return;
+ KASSERT(pcc->pcc_data == NULL);
+
+ /*
+ * Otherwise, allocate storage and, while the constructor list
+ * is locked, run constructors for all percpus on this CPU.
+ */
pcc->pcc_size = size;
if (size) {
pcc->pcc_data = kmem_zalloc(pcc->pcc_size, KM_SLEEP);
+ mutex_enter(&percpu_allocation.lock);
+ while (percpu_allocation.busy)
+ cv_wait(&percpu_allocation.cv,
+ &percpu_allocation.lock);
+ percpu_allocation.busy = curlwp;
+ LIST_FOREACH(pc, &percpu_allocation.ctor_list, pc_list) {
+ KASSERT(pc->pc_ctor);
+ mutex_exit(&percpu_allocation.lock);
+ (*pc->pc_ctor)((char *)pcc->pcc_data + pc->pc_offset,
+ pc->pc_cookie, ci);
+ mutex_enter(&percpu_allocation.lock);
+ }
+ KASSERT(percpu_allocation.busy == curlwp);
+ percpu_allocation.busy = NULL;
+ cv_broadcast(&percpu_allocation.cv);
+ mutex_exit(&percpu_allocation.lock);
}
}
@@ -281,6 +321,7 @@ percpu_create(size_t size, percpu_callba
pc = kmem_alloc(sizeof(*pc), KM_SLEEP);
pc->pc_offset = offset;
pc->pc_size = size;
+ pc->pc_ctor = ctor;
pc->pc_dtor = dtor;
pc->pc_cookie = cookie;
@@ -289,6 +330,22 @@ percpu_create(size_t size, percpu_callba
struct cpu_info *ci;
void *buf;
+ /*
+ * Wait until nobody is using the list of percpus with
+ * constructors.
+ */
+ mutex_enter(&percpu_allocation.lock);
+ while (percpu_allocation.busy)
+ cv_wait(&percpu_allocation.cv,
+ &percpu_allocation.lock);
+ percpu_allocation.busy = curlwp;
+ mutex_exit(&percpu_allocation.lock);
+
+ /*
+ * Run the constructor for all CPUs. We use a
+ * temporary buffer wo that we need not hold the
+ * percpu_swap_lock while running the constructor.
+ */
buf = kmem_alloc(size, KM_SLEEP);
for (CPU_INFO_FOREACH(cii, ci)) {
memset(buf, 0, size);
@@ -299,6 +356,19 @@ percpu_create(size_t size, percpu_callba
}
explicit_memset(buf, 0, size);
kmem_free(buf, size);
+
+ /*
+ * Insert the percpu into the list of percpus with
+ * constructors. We are now done using the list, so it
+ * is safe for concurrent percpu_create or concurrent
+ * percpu_init_cpu to run.
+ */
+ mutex_enter(&percpu_allocation.lock);
+ KASSERT(percpu_allocation.busy == curlwp);
+ percpu_allocation.busy = NULL;
+ cv_broadcast(&percpu_allocation.cv);
+ LIST_INSERT_HEAD(&percpu_allocation.ctor_list, pc, pc_list);
+ mutex_exit(&percpu_allocation.lock);
} else {
percpu_zero(pc, size);
}
@@ -320,6 +390,21 @@ percpu_free(percpu_t *pc, size_t size)
ASSERT_SLEEPABLE();
KASSERT(size == pc->pc_size);
+ /*
+ * If there's a constructor, take the percpu off the list of
+ * percpus with constructors, but first wait until nobody is
+ * using the list.
+ */
+ if (pc->pc_ctor) {
+ mutex_enter(&percpu_allocation.lock);
+ while (percpu_allocation.busy)
+ cv_wait(&percpu_allocation.cv,
+ &percpu_allocation.lock);
+ LIST_REMOVE(pc, pc_list);
+ mutex_exit(&percpu_allocation.lock);
+ }
+
+ /* If there's a destructor, run it now for all CPUs. */
if (pc->pc_dtor) {
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
