Module Name: src
Committed By: ad
Date: Sun Jun 14 21:34:25 UTC 2020
Modified Files:
src/sys/kern: subr_pool.c
src/sys/sys: pool.h
src/usr.bin/vmstat: vmstat.c
Log Message:
pool_cache:
- make all counters per-CPU and make cache layer do its work with atomic ops.
- conserve memory by caching empty groups globally.
To generate a diff of this commit:
cvs rdiff -u -r1.270 -r1.271 src/sys/kern/subr_pool.c
cvs rdiff -u -r1.90 -r1.91 src/sys/sys/pool.h
cvs rdiff -u -r1.240 -r1.241 src/usr.bin/vmstat/vmstat.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/subr_pool.c
diff -u src/sys/kern/subr_pool.c:1.270 src/sys/kern/subr_pool.c:1.271
--- src/sys/kern/subr_pool.c:1.270 Sun Jun 7 09:45:19 2020
+++ src/sys/kern/subr_pool.c Sun Jun 14 21:34:25 2020
@@ -1,8 +1,8 @@
-/* $NetBSD: subr_pool.c,v 1.270 2020/06/07 09:45:19 maxv Exp $ */
+/* $NetBSD: subr_pool.c,v 1.271 2020/06/14 21:34:25 ad Exp $ */
/*
- * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015, 2018
- * The NetBSD Foundation, Inc.
+ * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015, 2018,
+ * 2020 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -33,7 +33,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.270 2020/06/07 09:45:19 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.271 2020/06/14 21:34:25 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_ddb.h"
@@ -52,6 +52,7 @@ __KERNEL_RCSID(0, "$NetBSD: subr_pool.c,
#include <sys/pool.h>
#include <sys/syslog.h>
#include <sys/debug.h>
+#include <sys/lock.h>
#include <sys/lockdebug.h>
#include <sys/xcall.h>
#include <sys/cpu.h>
@@ -337,6 +338,9 @@ static struct pool pcg_large_pool;
static struct pool cache_pool;
static struct pool cache_cpu_pool;
+static pcg_t *volatile pcg_large_cache __cacheline_aligned;
+static pcg_t *volatile pcg_normal_cache __cacheline_aligned;
+
/* List of all caches. */
TAILQ_HEAD(,pool_cache) pool_cache_head =
TAILQ_HEAD_INITIALIZER(pool_cache_head);
@@ -344,14 +348,17 @@ TAILQ_HEAD(,pool_cache) pool_cache_head
int pool_cache_disable; /* global disable for caching */
static const pcg_t pcg_dummy; /* zero sized: always empty, yet always full */
-static bool pool_cache_put_slow(pool_cache_cpu_t *, int,
+static bool pool_cache_put_slow(pool_cache_t, pool_cache_cpu_t *, int,
void *);
-static bool pool_cache_get_slow(pool_cache_cpu_t *, int,
+static bool pool_cache_get_slow(pool_cache_t, pool_cache_cpu_t *, int,
void **, paddr_t *, int);
static void pool_cache_cpu_init1(struct cpu_info *, pool_cache_t);
-static void pool_cache_invalidate_groups(pool_cache_t, pcg_t *);
+static int pool_cache_invalidate_groups(pool_cache_t, pcg_t *);
static void pool_cache_invalidate_cpu(pool_cache_t, u_int);
static void pool_cache_transfer(pool_cache_t);
+static int pool_pcg_get(pcg_t *volatile *, pcg_t **);
+static int pool_pcg_put(pcg_t *volatile *, pcg_t *);
+static pcg_t * pool_pcg_trunc(pcg_t *volatile *);
static int pool_catchup(struct pool *);
static void pool_prime_page(struct pool *, void *,
@@ -1804,7 +1811,8 @@ pool_print1(struct pool *pp, const char
pool_cache_t pc;
pcg_t *pcg;
pool_cache_cpu_t *cc;
- uint64_t cpuhit, cpumiss;
+ uint64_t cpuhit, cpumiss, pchit, pcmiss;
+ uint32_t nfull;
int i, print_log = 0, print_pagelist = 0, print_cache = 0;
char c;
@@ -1881,31 +1889,26 @@ pool_print1(struct pool *pp, const char
if (pc != NULL) {
cpuhit = 0;
cpumiss = 0;
+ pcmiss = 0;
+ nfull = 0;
for (i = 0; i < __arraycount(pc->pc_cpus); i++) {
if ((cc = pc->pc_cpus[i]) == NULL)
continue;
cpuhit += cc->cc_hits;
cpumiss += cc->cc_misses;
+ pcmiss += cc->cc_pcmisses;
+ nfull += cc->cc_nfull;
}
+ pchit = cpumiss - pcmiss;
(*pr)("\tcpu layer hits %llu misses %llu\n", cpuhit, cpumiss);
- (*pr)("\tcache layer hits %llu misses %llu\n",
- pc->pc_hits, pc->pc_misses);
- (*pr)("\tcache layer entry uncontended %llu contended %llu\n",
- pc->pc_hits + pc->pc_misses - pc->pc_contended,
- pc->pc_contended);
- (*pr)("\tcache layer empty groups %u full groups %u\n",
- pc->pc_nempty, pc->pc_nfull);
+ (*pr)("\tcache layer hits %llu misses %llu\n", pchit, pcmiss);
+ (*pr)("\tcache layer full groups %u\n", nfull);
if (print_cache) {
(*pr)("\tfull cache groups:\n");
for (pcg = pc->pc_fullgroups; pcg != NULL;
pcg = pcg->pcg_next) {
PR_GROUPLIST(pcg);
}
- (*pr)("\tempty cache groups:\n");
- for (pcg = pc->pc_emptygroups; pcg != NULL;
- pcg = pcg->pcg_next) {
- PR_GROUPLIST(pcg);
- }
}
}
#undef PR_GROUPLIST
@@ -2051,7 +2054,6 @@ pool_cache_bootstrap(pool_cache_t pc, si
palloc = &pool_allocator_nointr;
}
pool_init(pp, size, align, align_offset, flags, wchan, palloc, ipl);
- mutex_init(&pc->pc_lock, MUTEX_DEFAULT, ipl);
if (ctor == NULL) {
ctor = NO_CTOR;
@@ -2060,27 +2062,22 @@ pool_cache_bootstrap(pool_cache_t pc, si
dtor = NO_DTOR;
}
- pc->pc_emptygroups = NULL;
pc->pc_fullgroups = NULL;
pc->pc_partgroups = NULL;
pc->pc_ctor = ctor;
pc->pc_dtor = dtor;
pc->pc_arg = arg;
- pc->pc_hits = 0;
- pc->pc_misses = 0;
- pc->pc_nempty = 0;
- pc->pc_npart = 0;
- pc->pc_nfull = 0;
- pc->pc_contended = 0;
pc->pc_refcnt = 0;
pc->pc_freecheck = NULL;
if ((flags & PR_LARGECACHE) != 0) {
pc->pc_pcgsize = PCG_NOBJECTS_LARGE;
pc->pc_pcgpool = &pcg_large_pool;
+ pc->pc_pcgcache = &pcg_large_cache;
} else {
pc->pc_pcgsize = PCG_NOBJECTS_NORMAL;
pc->pc_pcgpool = &pcg_normal_pool;
+ pc->pc_pcgcache = &pcg_normal_cache;
}
/* Allocate per-CPU caches. */
@@ -2157,7 +2154,6 @@ pool_cache_bootstrap_destroy(pool_cache_
pool_cache_invalidate_cpu(pc, i);
/* Finally, destroy it. */
- mutex_destroy(&pc->pc_lock);
pool_destroy(pp);
}
@@ -2177,7 +2173,6 @@ pool_cache_cpu_init1(struct cpu_info *ci
KASSERT(index < __arraycount(pc->pc_cpus));
if ((cc = pc->pc_cpus[index]) != NULL) {
- KASSERT(cc->cc_cpuindex == index);
return;
}
@@ -2189,20 +2184,19 @@ pool_cache_cpu_init1(struct cpu_info *ci
cc = &pc->pc_cpu0;
pc->pc_ncpu = 1;
} else {
- mutex_enter(&pc->pc_lock);
pc->pc_ncpu++;
- mutex_exit(&pc->pc_lock);
cc = pool_get(&cache_cpu_pool, PR_WAITOK);
}
- cc->cc_ipl = pc->pc_pool.pr_ipl;
- cc->cc_iplcookie = makeiplcookie(cc->cc_ipl);
- cc->cc_cache = pc;
- cc->cc_cpuindex = index;
- cc->cc_hits = 0;
- cc->cc_misses = 0;
cc->cc_current = __UNCONST(&pcg_dummy);
cc->cc_previous = __UNCONST(&pcg_dummy);
+ cc->cc_pcgcache = pc->pc_pcgcache;
+ cc->cc_hits = 0;
+ cc->cc_misses = 0;
+ cc->cc_pcmisses = 0;
+ cc->cc_contended = 0;
+ cc->cc_nfull = 0;
+ cc->cc_npart = 0;
pc->pc_cpus[index] = cc;
}
@@ -2268,16 +2262,17 @@ pool_cache_destruct_object(pool_cache_t
/*
* pool_cache_invalidate_groups:
*
- * Invalidate a chain of groups and destruct all objects.
+ * Invalidate a chain of groups and destruct all objects. Return the
+ * number of groups that were invalidated.
*/
-static void
+static int
pool_cache_invalidate_groups(pool_cache_t pc, pcg_t *pcg)
{
void *object;
pcg_t *next;
- int i;
+ int i, n;
- for (; pcg != NULL; pcg = next) {
+ for (n = 0; pcg != NULL; pcg = next, n++) {
next = pcg->pcg_next;
for (i = 0; i < pcg->pcg_avail; i++) {
@@ -2292,6 +2287,7 @@ pool_cache_invalidate_groups(pool_cache_
pool_put(&pcg_normal_pool, pcg);
}
}
+ return n;
}
/*
@@ -2311,7 +2307,8 @@ void
pool_cache_invalidate(pool_cache_t pc)
{
uint64_t where;
- pcg_t *full, *empty, *part;
+ pcg_t *pcg;
+ int n, s;
KASSERT(!cpu_intr_p() && !cpu_softintr_p());
@@ -2335,22 +2332,24 @@ pool_cache_invalidate(pool_cache_t pc)
xc_wait(where);
}
- /* Empty pool caches, then invalidate objects */
- mutex_enter(&pc->pc_lock);
- full = pc->pc_fullgroups;
- empty = pc->pc_emptygroups;
- part = pc->pc_partgroups;
- pc->pc_fullgroups = NULL;
- pc->pc_emptygroups = NULL;
- pc->pc_partgroups = NULL;
- pc->pc_nfull = 0;
- pc->pc_nempty = 0;
- pc->pc_npart = 0;
- mutex_exit(&pc->pc_lock);
-
- pool_cache_invalidate_groups(pc, full);
- pool_cache_invalidate_groups(pc, empty);
- pool_cache_invalidate_groups(pc, part);
+ /* Now dequeue and invalidate everything. */
+ pcg = pool_pcg_trunc(&pcg_normal_cache);
+ (void)pool_cache_invalidate_groups(pc, pcg);
+
+ pcg = pool_pcg_trunc(&pcg_large_cache);
+ (void)pool_cache_invalidate_groups(pc, pcg);
+
+ pcg = pool_pcg_trunc(&pc->pc_fullgroups);
+ n = pool_cache_invalidate_groups(pc, pcg);
+ s = splvm();
+ ((pool_cache_cpu_t *)pc->pc_cpus[curcpu()->ci_index])->cc_nfull -= n;
+ splx(s);
+
+ pcg = pool_pcg_trunc(&pc->pc_partgroups);
+ n = pool_cache_invalidate_groups(pc, pcg);
+ s = splvm();
+ ((pool_cache_cpu_t *)pc->pc_cpus[curcpu()->ci_index])->cc_npart -= n;
+ splx(s);
}
/*
@@ -2421,61 +2420,131 @@ pool_cache_prime(pool_cache_t pc, int n)
pool_prime(&pc->pc_pool, n);
}
+/*
+ * pool_pcg_get:
+ *
+ * Get a cache group from the specified list. Return true if
+ * contention was encountered. Must be called at IPL_VM because
+ * of spin wait vs. kernel_lock.
+ */
+static int
+pool_pcg_get(pcg_t *volatile *head, pcg_t **pcgp)
+{
+ int count = SPINLOCK_BACKOFF_MIN;
+ pcg_t *o, *n;
+
+ for (o = atomic_load_relaxed(head);; o = n) {
+ if (__predict_false(o == &pcg_dummy)) {
+ /* Wait for concurrent get to complete. */
+ SPINLOCK_BACKOFF(count);
+ n = atomic_load_relaxed(head);
+ continue;
+ }
+ if (__predict_false(o == NULL)) {
+ break;
+ }
+ /* Lock out concurrent get/put. */
+ n = atomic_cas_ptr(head, o, __UNCONST(&pcg_dummy));
+ if (o == n) {
+ /* Fetch pointer to next item and then unlock. */
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_datadep_consumer(); /* alpha */
+#endif
+ n = atomic_load_relaxed(&o->pcg_next);
+ atomic_store_release(head, n);
+ break;
+ }
+ }
+ *pcgp = o;
+ return count != SPINLOCK_BACKOFF_MIN;
+}
+
+/*
+ * pool_pcg_trunc:
+ *
+ * Chop out entire list of pool cache groups.
+ */
+static pcg_t *
+pool_pcg_trunc(pcg_t *volatile *head)
+{
+ int count = SPINLOCK_BACKOFF_MIN, s;
+ pcg_t *o, *n;
+
+ s = splvm();
+ for (o = atomic_load_relaxed(head);; o = n) {
+ if (__predict_false(o == &pcg_dummy)) {
+ /* Wait for concurrent get to complete. */
+ SPINLOCK_BACKOFF(count);
+ n = atomic_load_relaxed(head);
+ continue;
+ }
+ n = atomic_cas_ptr(head, o, NULL);
+ if (o == n) {
+ splx(s);
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_datadep_consumer(); /* alpha */
+#endif
+ return o;
+ }
+ }
+}
+
+/*
+ * pool_pcg_put:
+ *
+ * Put a pool cache group to the specified list. Return true if
+ * contention was encountered. Must be called at IPL_VM because of
+ * spin wait vs. kernel_lock.
+ */
+static int
+pool_pcg_put(pcg_t *volatile *head, pcg_t *pcg)
+{
+ int count = SPINLOCK_BACKOFF_MIN;
+ pcg_t *o, *n;
+
+ for (o = atomic_load_relaxed(head);; o = n) {
+ if (__predict_false(o == &pcg_dummy)) {
+ /* Wait for concurrent get to complete. */
+ SPINLOCK_BACKOFF(count);
+ n = atomic_load_relaxed(head);
+ continue;
+ }
+ pcg->pcg_next = o;
+#ifndef __HAVE_ATOMIC_AS_MEMBAR
+ membar_exit();
+#endif
+ n = atomic_cas_ptr(head, o, pcg);
+ if (o == n) {
+ return count != SPINLOCK_BACKOFF_MIN;
+ }
+ }
+}
+
static bool __noinline
-pool_cache_get_slow(pool_cache_cpu_t *cc, int s, void **objectp,
- paddr_t *pap, int flags)
+pool_cache_get_slow(pool_cache_t pc, pool_cache_cpu_t *cc, int s,
+ void **objectp, paddr_t *pap, int flags)
{
pcg_t *pcg, *cur;
- uint64_t ncsw;
- pool_cache_t pc;
void *object;
KASSERT(cc->cc_current->pcg_avail == 0);
KASSERT(cc->cc_previous->pcg_avail == 0);
- pc = cc->cc_cache;
cc->cc_misses++;
/*
- * Nothing was available locally. Try and grab a group
- * from the cache.
+ * If there's a full group, release our empty group back to the
+ * cache. Install the full group as cc_current and return.
*/
- if (__predict_false(!mutex_tryenter(&pc->pc_lock))) {
- ncsw = curlwp->l_ncsw;
- __insn_barrier();
- mutex_enter(&pc->pc_lock);
- pc->pc_contended++;
-
- /*
- * If we context switched while locking, then
- * our view of the per-CPU data is invalid:
- * retry.
- */
- __insn_barrier();
- if (curlwp->l_ncsw != ncsw) {
- mutex_exit(&pc->pc_lock);
- return true;
- }
- }
-
- if (__predict_true((pcg = pc->pc_fullgroups) != NULL)) {
- /*
- * If there's a full group, release our empty
- * group back to the cache. Install the full
- * group as cc_current and return.
- */
+ cc->cc_contended += pool_pcg_get(&pc->pc_fullgroups, &pcg);
+ if (__predict_true(pcg != NULL)) {
+ KASSERT(pcg->pcg_avail == pcg->pcg_size);
if (__predict_true((cur = cc->cc_current) != &pcg_dummy)) {
KASSERT(cur->pcg_avail == 0);
- cur->pcg_next = pc->pc_emptygroups;
- pc->pc_emptygroups = cur;
- pc->pc_nempty++;
+ (void)pool_pcg_put(cc->cc_pcgcache, cur);
}
- KASSERT(pcg->pcg_avail == pcg->pcg_size);
+ cc->cc_nfull--;
cc->cc_current = pcg;
- pc->pc_fullgroups = pcg->pcg_next;
- pc->pc_hits++;
- pc->pc_nfull--;
- mutex_exit(&pc->pc_lock);
return true;
}
@@ -2484,8 +2553,7 @@ pool_cache_get_slow(pool_cache_cpu_t *cc
* path: fetch a new object from the pool and construct
* it.
*/
- pc->pc_misses++;
- mutex_exit(&pc->pc_lock);
+ cc->cc_pcmisses++;
splx(s);
object = pool_get(&pc->pc_pool, flags);
@@ -2549,7 +2617,6 @@ pool_cache_get_paddr(pool_cache_t pc, in
while (/* CONSTCOND */ true) {
/* Try and allocate an object from the current group. */
cc = pc->pc_cpus[curcpu()->ci_index];
- KASSERT(cc->cc_cache == pc);
pcg = cc->cc_current;
if (__predict_true(pcg->pcg_avail > 0)) {
object = pcg->pcg_objects[--pcg->pcg_avail].pcgo_va;
@@ -2585,7 +2652,7 @@ pool_cache_get_paddr(pool_cache_t pc, in
* no more objects are available, it will return false.
* Otherwise, we need to retry.
*/
- if (!pool_cache_get_slow(cc, s, &object, pap, flags)) {
+ if (!pool_cache_get_slow(pc, cc, s, &object, pap, flags)) {
if (object != NULL) {
kmsan_orig(object, pc->pc_pool.pr_size,
KMSAN_TYPE_POOL, __RET_ADDR);
@@ -2603,77 +2670,33 @@ pool_cache_get_paddr(pool_cache_t pc, in
}
static bool __noinline
-pool_cache_put_slow(pool_cache_cpu_t *cc, int s, void *object)
+pool_cache_put_slow(pool_cache_t pc, pool_cache_cpu_t *cc, int s, void *object)
{
- struct lwp *l = curlwp;
pcg_t *pcg, *cur;
- uint64_t ncsw;
- pool_cache_t pc;
KASSERT(cc->cc_current->pcg_avail == cc->cc_current->pcg_size);
KASSERT(cc->cc_previous->pcg_avail == cc->cc_previous->pcg_size);
- pc = cc->cc_cache;
- pcg = NULL;
cc->cc_misses++;
- ncsw = l->l_ncsw;
- __insn_barrier();
/*
- * If there are no empty groups in the cache then allocate one
- * while still unlocked.
+ * Try to get an empty group from the cache. If there are no empty
+ * groups in the cache then allocate one.
*/
- if (__predict_false(pc->pc_emptygroups == NULL)) {
+ (void)pool_pcg_get(cc->cc_pcgcache, &pcg);
+ if (__predict_false(pcg == NULL)) {
if (__predict_true(!pool_cache_disable)) {
pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT);
}
- /*
- * If pool_get() blocked, then our view of
- * the per-CPU data is invalid: retry.
- */
- __insn_barrier();
- if (__predict_false(l->l_ncsw != ncsw)) {
- if (pcg != NULL) {
- pool_put(pc->pc_pcgpool, pcg);
- }
- return true;
- }
if (__predict_true(pcg != NULL)) {
pcg->pcg_avail = 0;
pcg->pcg_size = pc->pc_pcgsize;
}
}
- /* Lock the cache. */
- if (__predict_false(!mutex_tryenter(&pc->pc_lock))) {
- mutex_enter(&pc->pc_lock);
- pc->pc_contended++;
-
- /*
- * If we context switched while locking, then our view of
- * the per-CPU data is invalid: retry.
- */
- __insn_barrier();
- if (__predict_false(l->l_ncsw != ncsw)) {
- mutex_exit(&pc->pc_lock);
- if (pcg != NULL) {
- pool_put(pc->pc_pcgpool, pcg);
- }
- return true;
- }
- }
-
- /* If there are no empty groups in the cache then allocate one. */
- if (pcg == NULL && pc->pc_emptygroups != NULL) {
- pcg = pc->pc_emptygroups;
- pc->pc_emptygroups = pcg->pcg_next;
- pc->pc_nempty--;
- }
-
/*
- * If there's a empty group, release our full group back
- * to the cache. Install the empty group to the local CPU
- * and return.
+ * If there's a empty group, release our full group back to the
+ * cache. Install the empty group to the local CPU and return.
*/
if (pcg != NULL) {
KASSERT(pcg->pcg_avail == 0);
@@ -2683,14 +2706,12 @@ pool_cache_put_slow(pool_cache_cpu_t *cc
cur = cc->cc_current;
if (__predict_true(cur != &pcg_dummy)) {
KASSERT(cur->pcg_avail == cur->pcg_size);
- cur->pcg_next = pc->pc_fullgroups;
- pc->pc_fullgroups = cur;
- pc->pc_nfull++;
+ cc->cc_contended +=
+ pool_pcg_put(&pc->pc_fullgroups, cur);
+ cc->cc_nfull++;
}
cc->cc_current = pcg;
}
- pc->pc_hits++;
- mutex_exit(&pc->pc_lock);
return true;
}
@@ -2699,8 +2720,7 @@ pool_cache_put_slow(pool_cache_cpu_t *cc
* allocate an empty group. Take the slow path and destroy
* the object here and now.
*/
- pc->pc_misses++;
- mutex_exit(&pc->pc_lock);
+ cc->cc_pcmisses++;
splx(s);
pool_cache_destruct_object(pc, object);
@@ -2738,7 +2758,6 @@ pool_cache_put_paddr(pool_cache_t pc, vo
while (/* CONSTCOND */ true) {
/* If the current group isn't full, release it there. */
cc = pc->pc_cpus[curcpu()->ci_index];
- KASSERT(cc->cc_cache == pc);
pcg = cc->cc_current;
if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) {
pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object;
@@ -2765,7 +2784,7 @@ pool_cache_put_paddr(pool_cache_t pc, vo
* If put_slow() releases the object for us, it
* will return false. Otherwise we need to retry.
*/
- if (!pool_cache_put_slow(cc, s, object))
+ if (!pool_cache_put_slow(pc, cc, s, object))
break;
}
}
@@ -2780,11 +2799,10 @@ static void
pool_cache_transfer(pool_cache_t pc)
{
pool_cache_cpu_t *cc;
- pcg_t *prev, *cur, **list;
+ pcg_t *prev, *cur;
int s;
s = splvm();
- mutex_enter(&pc->pc_lock);
cc = pc->pc_cpus[curcpu()->ci_index];
cur = cc->cc_current;
cc->cc_current = __UNCONST(&pcg_dummy);
@@ -2792,33 +2810,26 @@ pool_cache_transfer(pool_cache_t pc)
cc->cc_previous = __UNCONST(&pcg_dummy);
if (cur != &pcg_dummy) {
if (cur->pcg_avail == cur->pcg_size) {
- list = &pc->pc_fullgroups;
- pc->pc_nfull++;
+ (void)pool_pcg_put(&pc->pc_fullgroups, cur);
+ cc->cc_nfull++;
} else if (cur->pcg_avail == 0) {
- list = &pc->pc_emptygroups;
- pc->pc_nempty++;
+ (void)pool_pcg_put(pc->pc_pcgcache, cur);
} else {
- list = &pc->pc_partgroups;
- pc->pc_npart++;
+ (void)pool_pcg_put(&pc->pc_partgroups, cur);
+ cc->cc_npart++;
}
- cur->pcg_next = *list;
- *list = cur;
}
if (prev != &pcg_dummy) {
if (prev->pcg_avail == prev->pcg_size) {
- list = &pc->pc_fullgroups;
- pc->pc_nfull++;
+ (void)pool_pcg_put(&pc->pc_fullgroups, prev);
+ cc->cc_nfull++;
} else if (prev->pcg_avail == 0) {
- list = &pc->pc_emptygroups;
- pc->pc_nempty++;
+ (void)pool_pcg_put(pc->pc_pcgcache, prev);
} else {
- list = &pc->pc_partgroups;
- pc->pc_npart++;
+ (void)pool_pcg_put(&pc->pc_partgroups, prev);
+ cc->cc_npart++;
}
- prev->pcg_next = *list;
- *list = prev;
}
- mutex_exit(&pc->pc_lock);
splx(s);
}
@@ -3325,31 +3336,32 @@ pool_sysctl(SYSCTLFN_ARGS)
data.pr_cache_nmiss_pcpu = 0;
data.pr_cache_nhit_pcpu = 0;
+ data.pr_cache_nmiss_global = 0;
+ data.pr_cache_nempty = 0;
+ data.pr_cache_ncontended = 0;
+ data.pr_cache_npartial = 0;
if (pp->pr_cache) {
+ uint32_t nfull = 0;
pc = pp->pr_cache;
data.pr_cache_meta_size = pc->pc_pcgsize;
- data.pr_cache_nfull = pc->pc_nfull;
- data.pr_cache_npartial = pc->pc_npart;
- data.pr_cache_nempty = pc->pc_nempty;
- data.pr_cache_ncontended = pc->pc_contended;
- data.pr_cache_nmiss_global = pc->pc_misses;
- data.pr_cache_nhit_global = pc->pc_hits;
for (i = 0; i < pc->pc_ncpu; ++i) {
cc = pc->pc_cpus[i];
if (cc == NULL)
continue;
+ data.pr_cache_ncontended += cc->cc_contended;
data.pr_cache_nmiss_pcpu += cc->cc_misses;
data.pr_cache_nhit_pcpu += cc->cc_hits;
+ data.pr_cache_nmiss_global += cc->cc_pcmisses;
+ nfull += cc->cc_nfull; /* 32-bit rollover! */
+ data.pr_cache_npartial = cc->cc_npart;
}
+ data.pr_cache_nfull = nfull;
} else {
data.pr_cache_meta_size = 0;
data.pr_cache_nfull = 0;
- data.pr_cache_npartial = 0;
- data.pr_cache_nempty = 0;
- data.pr_cache_ncontended = 0;
- data.pr_cache_nmiss_global = 0;
- data.pr_cache_nhit_global = 0;
}
+ data.pr_cache_nhit_global = data.pr_cache_nmiss_pcpu -
+ data.pr_cache_nmiss_global;
error = sysctl_copyout(l, &data, oldp, sizeof(data));
if (error)
Index: src/sys/sys/pool.h
diff -u src/sys/sys/pool.h:1.90 src/sys/sys/pool.h:1.91
--- src/sys/sys/pool.h:1.90 Mon Apr 13 00:27:17 2020
+++ src/sys/sys/pool.h Sun Jun 14 21:34:25 2020
@@ -1,7 +1,8 @@
-/* $NetBSD: pool.h,v 1.90 2020/04/13 00:27:17 chs Exp $ */
+/* $NetBSD: pool.h,v 1.91 2020/06/14 21:34:25 ad Exp $ */
/*-
- * Copyright (c) 1997, 1998, 1999, 2000, 2007 The NetBSD Foundation, Inc.
+ * Copyright (c) 1997, 1998, 1999, 2000, 2007, 2020
+ * The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -216,7 +217,7 @@ struct pool {
/*
* Cache group sizes, assuming 4-byte paddr_t on !_LP64.
- * All groups will be aligned to CACHE_LINE_SIZE.
+ * All groups will be aligned to COHERENCY_UNIT.
*/
#ifdef _LP64
#define PCG_NOBJECTS_NORMAL 15 /* 256 byte group */
@@ -239,17 +240,17 @@ typedef struct pool_cache_group {
pcgpair_t pcg_objects[1]; /* the objects */
} pcg_t;
+/* Pool cache CPU. Sized to 64 bytes on _LP64. */
typedef struct pool_cache_cpu {
- uint64_t cc_misses;
- uint64_t cc_hits;
struct pool_cache_group *cc_current;
struct pool_cache_group *cc_previous;
- struct pool_cache *cc_cache;
- int cc_ipl;
- int cc_cpuindex;
-#ifdef _KERNEL
- ipl_cookie_t cc_iplcookie;
-#endif
+ pcg_t *volatile *cc_pcgcache;
+ uint64_t cc_misses;
+ uint64_t cc_hits;
+ uint64_t cc_pcmisses;
+ uint64_t cc_contended;
+ uint32_t cc_nfull;
+ uint32_t cc_npart;
} pool_cache_cpu_t;
struct pool_cache {
@@ -257,34 +258,30 @@ struct pool_cache {
struct pool pc_pool;
/* Cache layer. */
- kmutex_t pc_lock; /* locks cache layer */
TAILQ_ENTRY(pool_cache)
pc_cachelist; /* entry on global cache list */
- pcg_t *pc_emptygroups;/* list of empty cache groups */
- pcg_t *pc_fullgroups; /* list of full cache groups */
- pcg_t *pc_partgroups; /* groups for reclamation */
struct pool *pc_pcgpool; /* Pool of cache groups */
+ pcg_t *volatile *pc_pcgcache; /* list of empty cache groups */
int pc_pcgsize; /* Use large cache groups? */
int pc_ncpu; /* number cpus set up */
int (*pc_ctor)(void *, void *, int);
void (*pc_dtor)(void *, void *);
void *pc_arg; /* for ctor/ctor */
- uint64_t pc_hits; /* cache layer hits */
- uint64_t pc_misses; /* cache layer misses */
- uint64_t pc_contended; /* contention events on cache */
- unsigned int pc_nempty; /* empty groups in cache */
- unsigned int pc_nfull; /* full groups in cache */
- unsigned int pc_npart; /* partial groups in cache */
unsigned int pc_refcnt; /* ref count for pagedaemon, etc */
+ void *pc_cpus[MAXCPUS];
/* Diagnostic aides. */
void *pc_freecheck;
bool pc_redzone;
size_t pc_reqsize;
- /* CPU layer. */
+ /* Hot items. */
+ pcg_t *volatile pc_fullgroups /* list of full cache groups */
+ __aligned(CACHE_LINE_SIZE);
+ pcg_t *volatile pc_partgroups; /* groups for reclamation */
+
+ /* Boot cpu. */
pool_cache_cpu_t pc_cpu0 __aligned(CACHE_LINE_SIZE);
- void *pc_cpus[MAXCPUS] __aligned(CACHE_LINE_SIZE);
};
#endif /* __POOL_EXPOSE */
Index: src/usr.bin/vmstat/vmstat.c
diff -u src/usr.bin/vmstat/vmstat.c:1.240 src/usr.bin/vmstat/vmstat.c:1.241
--- src/usr.bin/vmstat/vmstat.c:1.240 Thu Jun 11 22:21:05 2020
+++ src/usr.bin/vmstat/vmstat.c Sun Jun 14 21:34:25 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vmstat.c,v 1.240 2020/06/11 22:21:05 ad Exp $ */
+/* $NetBSD: vmstat.c,v 1.241 2020/06/14 21:34:25 ad Exp $ */
/*-
* Copyright (c) 1998, 2000, 2001, 2007, 2019, 2020
@@ -71,7 +71,7 @@ __COPYRIGHT("@(#) Copyright (c) 1980, 19
#if 0
static char sccsid[] = "@(#)vmstat.c 8.2 (Berkeley) 3/1/95";
#else
-__RCSID("$NetBSD: vmstat.c,v 1.240 2020/06/11 22:21:05 ad Exp $");
+__RCSID("$NetBSD: vmstat.c,v 1.241 2020/06/14 21:34:25 ad Exp $");
#endif
#endif /* not lint */
@@ -1796,7 +1796,8 @@ dopoolcache(int verbose)
TAILQ_HEAD(,pool) pool_head;
struct pool pool, *pp = &pool;
char name[32];
- uint64_t cpuhit, cpumiss, tot;
+ uint64_t cpuhit, cpumiss, pchit, pcmiss, contended, tot;
+ uint32_t nfull;
void *addr;
int first, ovflw;
size_t i;
@@ -1815,12 +1816,13 @@ dopoolcache(int verbose)
deref_kptr(pp->pr_wchan, name, sizeof(name),
"pool wait channel trashed");
deref_kptr(pp->pr_cache, pc, sizeof(*pc), "pool cache trashed");
- if (pc->pc_misses == 0 && !verbose)
- continue;
name[sizeof(name)-1] = '\0';
cpuhit = 0;
cpumiss = 0;
+ pcmiss = 0;
+ contended = 0;
+ nfull = 0;
for (i = 0; i < __arraycount(pc->pc_cpus); i++) {
if ((addr = pc->pc_cpus[i]) == NULL)
continue;
@@ -1828,7 +1830,14 @@ dopoolcache(int verbose)
"pool cache cpu trashed");
cpuhit += cc->cc_hits;
cpumiss += cc->cc_misses;
+ pcmiss += cc->cc_pcmisses;
+ nfull += cc->cc_nfull;
+ contended += cc->cc_contended;
}
+ pchit = cpumiss - pcmiss;
+
+ if (pcmiss == 0 && !verbose)
+ continue;
if (first) {
(void)printf("Pool cache statistics.\n");
@@ -1849,14 +1858,14 @@ dopoolcache(int verbose)
ovflw = 0;
PRWORD(ovflw, "%-*s", 13, 1, name);
- PRWORD(ovflw, " %*llu", 6, 1, (long long)pc->pc_contended);
+ PRWORD(ovflw, " %*llu", 6, 1, (long long)contended);
PRWORD(ovflw, " %*u", 6, 1, pc->pc_pcgsize);
- PRWORD(ovflw, " %*u", 5, 1, pc->pc_nfull);
- PRWORD(ovflw, " %*u", 5, 1, pc->pc_nempty);
- PRWORD(ovflw, " %*llu", 10, 1, (long long)pc->pc_misses);
+ PRWORD(ovflw, " %*u", 5, 1, nfull);
+ PRWORD(ovflw, " %*u", 5, 1, 0);
+ PRWORD(ovflw, " %*llu", 10, 1, (long long)pcmiss);
- tot = pc->pc_hits + pc->pc_misses;
- p = pc->pc_hits * 100.0 / (tot);
+ tot = pchit + pcmiss;
+ p = pchit * 100.0 / (tot);
PRWORD(ovflw, " %*llu", 11, 1, (long long)tot);
PRWORD(ovflw, " %*.1f", 6, 1, p);
