Author: jeff Date: Thu Nov 28 00:19:09 2019 New Revision: 355149 URL: https://svnweb.freebsd.org/changeset/base/355149
Log: Implement a sysctl tree for uma zones to assist in debugging and provide more statistcs than are exported via the ABI stable vmstat interface. Rename uz_count to uz_bucket_size because even I was confused by the name after returning to the source years later. Reviewed by: rlibby Differential Revision: https://reviews.freebsd.org/D22554 Modified: head/sys/vm/uma_core.c head/sys/vm/uma_int.h Modified: head/sys/vm/uma_core.c ============================================================================== --- head/sys/vm/uma_core.c Wed Nov 27 23:19:06 2019 (r355148) +++ head/sys/vm/uma_core.c Thu Nov 28 00:19:09 2019 (r355149) @@ -119,6 +119,7 @@ static uma_zone_t hashzone; int uma_align_cache = 64 - 1; static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets"); +static MALLOC_DEFINE(M_UMA, "UMA", "UMA Misc"); /* * Are we allowed to allocate buckets? @@ -258,8 +259,8 @@ static void zone_dtor(void *, int, void *); static int zero_init(void *, int, int); static void keg_small_init(uma_keg_t keg); static void keg_large_init(uma_keg_t keg); -static void zone_foreach(void (*zfunc)(uma_zone_t)); -static void zone_timeout(uma_zone_t zone); +static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *); +static void zone_timeout(uma_zone_t zone, void *); static int hash_alloc(struct uma_hash *, u_int); static int hash_expand(struct uma_hash *, struct uma_hash *); static void hash_free(struct uma_hash *hash); @@ -285,10 +286,10 @@ static void uma_zero_item(void *, uma_zone_t); static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int); static bool cache_free(uma_zone_t, uma_cache_t, void *, void *, int); -void uma_print_zone(uma_zone_t); -void uma_print_stats(void); static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS); static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS); +static int sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS); +static int sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS); #ifdef INVARIANTS static bool uma_dbg_kskip(uma_keg_t keg, void *mem); @@ -296,6 +297,7 @@ static bool uma_dbg_zskip(uma_zone_t zone, void *mem); static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item); static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item); +SYSCTL_NODE(_vm, OID_AUTO, uma, CTLFLAG_RW, 0, "Universal Memory Allocator"); static SYSCTL_NODE(_vm, OID_AUTO, debug, CTLFLAG_RD, 0, "Memory allocation debugging"); @@ -458,7 +460,7 @@ bucket_alloc(uma_zone_t zone, void *udata, int flags) } if ((uintptr_t)udata & UMA_ZFLAG_CACHEONLY) flags |= M_NOVM; - ubz = bucket_zone_lookup(zone->uz_count); + ubz = bucket_zone_lookup(zone->uz_bucket_size); if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0) ubz++; bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags); @@ -575,7 +577,7 @@ static void uma_timeout(void *unused) { bucket_enable(); - zone_foreach(zone_timeout); + zone_foreach(zone_timeout, NULL); /* Reschedule this event */ callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); @@ -605,7 +607,7 @@ zone_domain_update_wss(uma_zone_domain_t zdom) * Returns nothing. */ static void -zone_timeout(uma_zone_t zone) +zone_timeout(uma_zone_t zone, void *unused) { uma_keg_t keg; u_int slabs; @@ -845,19 +847,20 @@ cache_drain(uma_zone_t zone) } static void -cache_shrink(uma_zone_t zone) +cache_shrink(uma_zone_t zone, void *unused) { if (zone->uz_flags & UMA_ZFLAG_INTERNAL) return; ZONE_LOCK(zone); - zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2; + zone->uz_bucket_size = + (zone->uz_bucket_size_min + zone->uz_bucket_size) / 2; ZONE_UNLOCK(zone); } static void -cache_drain_safe_cpu(uma_zone_t zone) +cache_drain_safe_cpu(uma_zone_t zone, void *unused) { uma_cache_t cache; uma_bucket_t b1, b2, b3; @@ -920,9 +923,9 @@ pcpu_cache_drain_safe(uma_zone_t zone) * Polite bucket sizes shrinking was not enouth, shrink aggressively. */ if (zone) - cache_shrink(zone); + cache_shrink(zone, NULL); else - zone_foreach(cache_shrink); + zone_foreach(cache_shrink, NULL); CPU_FOREACH(cpu) { thread_lock(curthread); @@ -930,9 +933,9 @@ pcpu_cache_drain_safe(uma_zone_t zone) thread_unlock(curthread); if (zone) - cache_drain_safe_cpu(zone); + cache_drain_safe_cpu(zone, NULL); else - zone_foreach(cache_drain_safe_cpu); + zone_foreach(cache_drain_safe_cpu, NULL); } thread_lock(curthread); sched_unbind(curthread); @@ -991,8 +994,8 @@ bucket_cache_reclaim(uma_zone_t zone, bool drain) * Shrink the zone bucket size to ensure that the per-CPU caches * don't grow too large. */ - if (zone->uz_count > zone->uz_count_min) - zone->uz_count--; + if (zone->uz_bucket_size > zone->uz_bucket_size_min) + zone->uz_bucket_size--; } static void @@ -1121,14 +1124,14 @@ out: } static void -zone_drain(uma_zone_t zone) +zone_drain(uma_zone_t zone, void *unused) { zone_reclaim(zone, M_NOWAIT, true); } static void -zone_trim(uma_zone_t zone) +zone_trim(uma_zone_t zone, void *unused) { zone_reclaim(zone, M_NOWAIT, false); @@ -1818,7 +1821,7 @@ keg_ctor(void *mem, int size, void *udata, int flags) } static void -zone_alloc_counters(uma_zone_t zone) +zone_alloc_counters(uma_zone_t zone, void *unused) { zone->uz_allocs = counter_u64_alloc(M_WAITOK); @@ -1826,6 +1829,169 @@ zone_alloc_counters(uma_zone_t zone) zone->uz_fails = counter_u64_alloc(M_WAITOK); } +#define UMA_MAX_DUP 999 +static void +zone_alloc_sysctl(uma_zone_t zone, void *unused) +{ + uma_zone_domain_t zdom; + uma_keg_t keg; + struct sysctl_oid *oid, *domainoid; + int domains, i; + static const char *nokeg = "cache zone"; + char *c; + + /* + * Make a sysctl safe copy of the zone name by removing + * any special characters and handling dups by appending + * an index. + */ + if (zone->uz_namecnt != 0) { + if (zone->uz_namecnt > UMA_MAX_DUP) + zone->uz_namecnt = UMA_MAX_DUP; + zone->uz_ctlname = malloc(strlen(zone->uz_name) + + sizeof(__XSTRING(UMA_MAX_DUP)) + 1 , M_UMA, M_WAITOK); + sprintf(zone->uz_ctlname, "%s_%d", zone->uz_name, + zone->uz_namecnt); + } else + zone->uz_ctlname = strdup(zone->uz_name, M_UMA); + for (c = zone->uz_ctlname; *c != '\0'; c++) + if (strchr("./\\ -", *c) != NULL) + *c = '_'; + + /* + * Basic parameters at the root. + */ + zone->uz_oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_vm_uma), + OID_AUTO, zone->uz_ctlname, CTLFLAG_RD, NULL, ""); + oid = zone->uz_oid; + SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "size", CTLFLAG_RD, &zone->uz_size, 0, "Allocation size"); + SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "flags", CTLFLAG_RD, &zone->uz_flags, 0, + "Allocator configuration flags"); + SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "bucket_size", CTLFLAG_RD, &zone->uz_bucket_size, 0, + "Desired per-cpu cache size"); + SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "bucket_size_max", CTLFLAG_RD, &zone->uz_bucket_size_max, 0, + "Maximum allowed per-cpu cache size"); + + /* + * keg if present. + */ + oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, + "keg", CTLFLAG_RD, NULL, ""); + keg = zone->uz_keg; + if ((zone->uz_flags & UMA_ZFLAG_CACHEONLY) == 0) { + SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "name", CTLFLAG_RD, keg->uk_name, "Keg name"); + SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "rsize", CTLFLAG_RD, &keg->uk_rsize, 0, + "Real object size with alignment"); + SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "ppera", CTLFLAG_RD, &keg->uk_ppera, 0, + "pages per-slab allocation"); + SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "ipers", CTLFLAG_RD, &keg->uk_ipers, 0, + "items available per-slab"); + SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "align", CTLFLAG_RD, &keg->uk_align, 0, + "item alignment mask"); + SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "pages", CTLFLAG_RD, &keg->uk_pages, 0, + "Total pages currently allocated from VM"); + SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "free", CTLFLAG_RD, &keg->uk_free, 0, + "items free in the slab layer"); + } else + SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "name", CTLFLAG_RD, nokeg, "Keg name"); + + /* + * Information about zone limits. + */ + oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, + "limit", CTLFLAG_RD, NULL, ""); + SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "items", CTLFLAG_RD, &zone->uz_items, 0, + "current number of cached items"); + SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "max_items", CTLFLAG_RD, &zone->uz_max_items, 0, + "Maximum number of cached items"); + SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0, + "Number of threads sleeping at limit"); + SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0, + "Total zone limit sleeps"); + + /* + * Per-domain information. + */ + if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) + domains = vm_ndomains; + else + domains = 1; + domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), + OID_AUTO, "domain", CTLFLAG_RD, NULL, ""); + for (i = 0; i < domains; i++) { + zdom = &zone->uz_domain[i]; + oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid), + OID_AUTO, VM_DOMAIN(i)->vmd_name, CTLFLAG_RD, NULL, ""); + SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "nitems", CTLFLAG_RD, &zdom->uzd_nitems, + "number of items in this domain"); + SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "imax", CTLFLAG_RD, &zdom->uzd_imax, + "maximum item count in this period"); + SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "imin", CTLFLAG_RD, &zdom->uzd_imin, + "minimum item count in this period"); + SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "wss", CTLFLAG_RD, &zdom->uzd_wss, + "Working set size"); + } + + /* + * General statistics. + */ + oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, + "stats", CTLFLAG_RD, NULL, ""); + SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "current", CTLFLAG_RD | CTLTYPE_INT | CTLFLAG_MPSAFE, + zone, 1, sysctl_handle_uma_zone_cur, "I", + "Current number of allocated items"); + SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "allocs", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE, + zone, 0, sysctl_handle_uma_zone_allocs, "QU", + "Total allocation calls"); + SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "frees", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE, + zone, 0, sysctl_handle_uma_zone_frees, "QU", + "Total free calls"); + SYSCTL_ADD_COUNTER_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "fails", CTLFLAG_RD, &zone->uz_fails, + "Number of allocation failures"); + SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "xdomain", CTLFLAG_RD, &zone->uz_xdomain, 0, + "Free calls from the wrong domain"); +} + +struct uma_zone_count { + const char *name; + int count; +}; + +static void +zone_count(uma_zone_t zone, void *arg) +{ + struct uma_zone_count *cnt; + + cnt = arg; + if (strcmp(zone->uz_name, cnt->name) == 0) + cnt->count++; +} + /* * Zone header ctor. This initializes all fields, locks, etc. * @@ -1835,6 +2001,7 @@ zone_alloc_counters(uma_zone_t zone) static int zone_ctor(void *mem, int size, void *udata, int flags) { + struct uma_zone_count cnt; struct uma_zctor_args *arg = udata; uma_zone_t zone = mem; uma_zone_t z; @@ -1849,9 +2016,9 @@ zone_ctor(void *mem, int size, void *udata, int flags) zone->uz_fini = NULL; zone->uz_sleeps = 0; zone->uz_xdomain = 0; - zone->uz_count = 0; - zone->uz_count_min = 0; - zone->uz_count_max = BUCKET_MAX; + zone->uz_bucket_size = 0; + zone->uz_bucket_size_min = 0; + zone->uz_bucket_size_max = BUCKET_MAX; zone->uz_flags = 0; zone->uz_warning = NULL; /* The domain structures follow the cpu structures. */ @@ -1859,13 +2026,11 @@ zone_ctor(void *mem, int size, void *udata, int flags) zone->uz_bkt_max = ULONG_MAX; timevalclear(&zone->uz_ratecheck); - if (__predict_true(booted == BOOT_RUNNING)) - zone_alloc_counters(zone); - else { - zone->uz_allocs = EARLY_COUNTER; - zone->uz_frees = EARLY_COUNTER; - zone->uz_fails = EARLY_COUNTER; - } + /* Count the number of duplicate names. */ + cnt.name = arg->name; + cnt.count = 0; + zone_foreach(zone_count, &cnt); + zone->uz_namecnt = cnt.count; for (i = 0; i < vm_ndomains; i++) TAILQ_INIT(&zone->uz_domain[i].uzd_buckets); @@ -1903,6 +2068,8 @@ zone_ctor(void *mem, int size, void *udata, int flags) keg = arg->keg; if (arg->flags & UMA_ZONE_SECONDARY) { + KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0, + ("Secondary zone requested UMA_ZFLAG_INTERNAL")); KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); zone->uz_init = arg->uminit; zone->uz_fini = arg->fini; @@ -1939,35 +2106,36 @@ zone_ctor(void *mem, int size, void *udata, int flags) return (error); } + /* Inherit properties from the keg. */ zone->uz_keg = keg; zone->uz_size = keg->uk_size; zone->uz_flags |= (keg->uk_flags & (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT)); - /* - * Some internal zones don't have room allocated for the per cpu - * caches. If we're internal, bail out here. - */ - if (keg->uk_flags & UMA_ZFLAG_INTERNAL) { - KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0, - ("Secondary zone requested UMA_ZFLAG_INTERNAL")); - return (0); +out: + if (__predict_true(booted == BOOT_RUNNING)) { + zone_alloc_counters(zone, NULL); + zone_alloc_sysctl(zone, NULL); + } else { + zone->uz_allocs = EARLY_COUNTER; + zone->uz_frees = EARLY_COUNTER; + zone->uz_fails = EARLY_COUNTER; } -out: KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) != (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET), ("Invalid zone flag combination")); - if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) { - zone->uz_count = BUCKET_MAX; - } else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0) { - zone->uz_count = BUCKET_MIN; - zone->uz_count_max = BUCKET_MIN; - } else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0) - zone->uz_count = 0; + if (arg->flags & UMA_ZFLAG_INTERNAL) + zone->uz_bucket_size_max = zone->uz_bucket_size = 0; + if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) + zone->uz_bucket_size = BUCKET_MAX; + else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0) + zone->uz_bucket_size_max = zone->uz_bucket_size = BUCKET_MIN; + else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0) + zone->uz_bucket_size = 0; else - zone->uz_count = bucket_select(zone->uz_size); - zone->uz_count_min = zone->uz_count; + zone->uz_bucket_size = bucket_select(zone->uz_size); + zone->uz_bucket_size_min = zone->uz_bucket_size; return (0); } @@ -2013,6 +2181,8 @@ zone_dtor(void *arg, int size, void *udata) zone = (uma_zone_t)arg; + sysctl_remove_oid(zone->uz_oid, 1, 1); + if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) cache_drain(zone); @@ -2039,6 +2209,7 @@ zone_dtor(void *arg, int size, void *udata) counter_u64_free(zone->uz_allocs); counter_u64_free(zone->uz_frees); counter_u64_free(zone->uz_fails); + free(zone->uz_ctlname, M_UMA); if (zone->uz_lockptr == &zone->uz_lock) ZONE_LOCK_FINI(zone); } @@ -2054,7 +2225,7 @@ zone_dtor(void *arg, int size, void *udata) * Nothing */ static void -zone_foreach(void (*zfunc)(uma_zone_t)) +zone_foreach(void (*zfunc)(uma_zone_t, void *arg), void *arg) { uma_keg_t keg; uma_zone_t zone; @@ -2068,10 +2239,10 @@ zone_foreach(void (*zfunc)(uma_zone_t)) rw_rlock(&uma_rwlock); LIST_FOREACH(keg, &uma_kegs, uk_link) { LIST_FOREACH(zone, &keg->uk_zones, uz_link) - zfunc(zone); + zfunc(zone, arg); } LIST_FOREACH(zone, &uma_cachezones, uz_link) - zfunc(zone); + zfunc(zone, arg); if (__predict_true(booted == BOOT_RUNNING)) rw_runlock(&uma_rwlock); } @@ -2249,7 +2420,8 @@ uma_startup3(void) uma_dbg_cnt = counter_u64_alloc(M_WAITOK); uma_skip_cnt = counter_u64_alloc(M_WAITOK); #endif - zone_foreach(zone_alloc_counters); + zone_foreach(zone_alloc_counters, NULL); + zone_foreach(zone_alloc_sysctl, NULL); callout_init(&uma_callout, 1); callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); booted = BOOT_RUNNING; @@ -2665,7 +2837,7 @@ cache_alloc(uma_zone_t zone, uma_cache_t cache, void * critical_enter(); /* Short-circuit for zones without buckets and low memory. */ - if (zone->uz_count == 0 || bucketdisable) + if (zone->uz_bucket_size == 0 || bucketdisable) return (false); cpu = curcpu; @@ -2702,8 +2874,8 @@ cache_alloc(uma_zone_t zone, uma_cache_t cache, void * * We bump the uz count when the cache size is insufficient to * handle the working set. */ - if (lockfail && zone->uz_count < zone->uz_count_max) - zone->uz_count++; + if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max) + zone->uz_bucket_size++; /* * Fill a bucket and attempt to use it as the alloc bucket. @@ -3005,11 +3177,11 @@ zone_alloc_bucket(uma_zone_t zone, void *udata, int do if (zone->uz_max_items > 0) { if (zone->uz_items >= zone->uz_max_items) return (false); - maxbucket = MIN(zone->uz_count, + maxbucket = MIN(zone->uz_bucket_size, zone->uz_max_items - zone->uz_items); zone->uz_items += maxbucket; } else - maxbucket = zone->uz_count; + maxbucket = zone->uz_bucket_size; ZONE_UNLOCK(zone); /* Don't wait for buckets, preserve caller's NOVM setting. */ @@ -3285,8 +3457,8 @@ zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, if (ZONE_TRYLOCK(zone) == 0) { /* Record contention to size the buckets. */ ZONE_LOCK(zone); - if (zone->uz_count < zone->uz_count_max) - zone->uz_count++; + if (zone->uz_bucket_size < zone->uz_bucket_size_max) + zone->uz_bucket_size++; } CTR3(KTR_UMA, @@ -3323,7 +3495,7 @@ cache_free(uma_zone_t zone, uma_cache_t cache, void *u CRITICAL_ASSERT(curthread); - if (zone->uz_count == 0 || bucketdisable) + if (zone->uz_bucket_size == 0 || bucketdisable) return false; cpu = curcpu; @@ -3523,9 +3695,9 @@ uma_zone_set_max(uma_zone_t zone, int nitems) ZONE_LOCK(zone); ubz = bucket_zone_max(zone, nitems); count = ubz != NULL ? ubz->ubz_entries : 0; - zone->uz_count_max = zone->uz_count = count; - if (zone->uz_count_min > zone->uz_count_max) - zone->uz_count_min = zone->uz_count_max; + zone->uz_bucket_size_max = zone->uz_bucket_size = count; + if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) + zone->uz_bucket_size_min = zone->uz_bucket_size_max; zone->uz_max_items = nitems; ZONE_UNLOCK(zone); @@ -3549,12 +3721,12 @@ uma_zone_set_maxcache(uma_zone_t zone, int nitems) bpcpu++; #endif nitems -= ubz->ubz_entries * bpcpu * mp_ncpus; - zone->uz_count_max = ubz->ubz_entries; + zone->uz_bucket_size_max = ubz->ubz_entries; } else { - zone->uz_count_max = zone->uz_count = 0; + zone->uz_bucket_size_max = zone->uz_bucket_size = 0; } - if (zone->uz_count_min > zone->uz_count_max) - zone->uz_count_min = zone->uz_count_max; + if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) + zone->uz_bucket_size_min = zone->uz_bucket_size_max; zone->uz_bkt_max = nitems; ZONE_UNLOCK(zone); } @@ -3602,20 +3774,71 @@ uma_zone_get_cur(uma_zone_t zone) ZONE_LOCK(zone); nitems = counter_u64_fetch(zone->uz_allocs) - counter_u64_fetch(zone->uz_frees); - CPU_FOREACH(i) { - /* - * See the comment in uma_vm_zone_stats() regarding the - * safety of accessing the per-cpu caches. With the zone lock - * held, it is safe, but can potentially result in stale data. - */ - nitems += zone->uz_cpu[i].uc_allocs - - zone->uz_cpu[i].uc_frees; + if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) { + CPU_FOREACH(i) { + /* + * See the comment in uma_vm_zone_stats() regarding + * the safety of accessing the per-cpu caches. With + * the zone lock held, it is safe, but can potentially + * result in stale data. + */ + nitems += zone->uz_cpu[i].uc_allocs - + zone->uz_cpu[i].uc_frees; + } } ZONE_UNLOCK(zone); return (nitems < 0 ? 0 : nitems); } +static uint64_t +uma_zone_get_allocs(uma_zone_t zone) +{ + uint64_t nitems; + u_int i; + + ZONE_LOCK(zone); + nitems = counter_u64_fetch(zone->uz_allocs); + if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) { + CPU_FOREACH(i) { + /* + * See the comment in uma_vm_zone_stats() regarding + * the safety of accessing the per-cpu caches. With + * the zone lock held, it is safe, but can potentially + * result in stale data. + */ + nitems += zone->uz_cpu[i].uc_allocs; + } + } + ZONE_UNLOCK(zone); + + return (nitems); +} + +static uint64_t +uma_zone_get_frees(uma_zone_t zone) +{ + uint64_t nitems; + u_int i; + + ZONE_LOCK(zone); + nitems = counter_u64_fetch(zone->uz_frees); + if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) { + CPU_FOREACH(i) { + /* + * See the comment in uma_vm_zone_stats() regarding + * the safety of accessing the per-cpu caches. With + * the zone lock held, it is safe, but can potentially + * result in stale data. + */ + nitems += zone->uz_cpu[i].uc_frees; + } + } + ZONE_UNLOCK(zone); + + return (nitems); +} + /* See uma.h */ void uma_zone_set_init(uma_zone_t zone, uma_init uminit) @@ -3800,14 +4023,14 @@ uma_reclaim(int req) switch (req) { case UMA_RECLAIM_TRIM: - zone_foreach(zone_trim); + zone_foreach(zone_trim, NULL); break; case UMA_RECLAIM_DRAIN: case UMA_RECLAIM_DRAIN_CPU: - zone_foreach(zone_drain); + zone_foreach(zone_drain, NULL); if (req == UMA_RECLAIM_DRAIN_CPU) { pcpu_cache_drain_safe(NULL); - zone_foreach(zone_drain); + zone_foreach(zone_drain, NULL); } break; default: @@ -3819,7 +4042,7 @@ uma_reclaim(int req) * we visit again so that we can free pages that are empty once other * zones are drained. We have to do the same for buckets. */ - zone_drain(slabzone); + zone_drain(slabzone, NULL); bucket_zone_drain(); sx_xunlock(&uma_reclaim_lock); } @@ -3859,14 +4082,14 @@ uma_zone_reclaim(uma_zone_t zone, int req) switch (req) { case UMA_RECLAIM_TRIM: - zone_trim(zone); + zone_trim(zone, NULL); break; case UMA_RECLAIM_DRAIN: - zone_drain(zone); + zone_drain(zone, NULL); break; case UMA_RECLAIM_DRAIN_CPU: pcpu_cache_drain_safe(zone); - zone_drain(zone); + zone_drain(zone, NULL); break; default: panic("unhandled reclamation request %d", req); @@ -3977,76 +4200,6 @@ uma_avail(void) return (uma_kmem_limit - uma_size()); } -void -uma_print_stats(void) -{ - zone_foreach(uma_print_zone); -} - -static void -slab_print(uma_slab_t slab) -{ - printf("slab: keg %p, data %p, freecount %d\n", - slab->us_keg, slab->us_data, slab->us_freecount); -} - -static void -cache_print(uma_cache_t cache) -{ - printf("alloc: %p(%d), free: %p(%d), cross: %p(%d)j\n", - cache->uc_allocbucket, - cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0, - cache->uc_freebucket, - cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0, - cache->uc_crossbucket, - cache->uc_crossbucket?cache->uc_crossbucket->ub_cnt:0); -} - -static void -uma_print_keg(uma_keg_t keg) -{ - uma_domain_t dom; - uma_slab_t slab; - int i; - - printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d " - "out %d free %d\n", - keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags, - keg->uk_ipers, keg->uk_ppera, - (keg->uk_pages / keg->uk_ppera) * keg->uk_ipers - keg->uk_free, - keg->uk_free); - for (i = 0; i < vm_ndomains; i++) { - dom = &keg->uk_domain[i]; - printf("Part slabs:\n"); - LIST_FOREACH(slab, &dom->ud_part_slab, us_link) - slab_print(slab); - printf("Free slabs:\n"); - LIST_FOREACH(slab, &dom->ud_free_slab, us_link) - slab_print(slab); - printf("Full slabs:\n"); - LIST_FOREACH(slab, &dom->ud_full_slab, us_link) - slab_print(slab); - } -} - -void -uma_print_zone(uma_zone_t zone) -{ - uma_cache_t cache; - int i; - - printf("zone: %s(%p) size %d maxitems %ju flags %#x\n", - zone->uz_name, zone, zone->uz_size, (uintmax_t)zone->uz_max_items, - zone->uz_flags); - if (zone->uz_lockptr != &zone->uz_lock) - uma_print_keg(zone->uz_keg); - CPU_FOREACH(i) { - cache = &zone->uz_cpu[i]; - printf("CPU %d Cache:\n", i); - cache_print(cache); - } -} - #ifdef DDB /* * Generate statistics across both the zone and its per-cpu cache's. Return @@ -4274,13 +4427,41 @@ sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS) int sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS) { - uma_zone_t zone = *(uma_zone_t *)arg1; + uma_zone_t zone; int cur; + /* + * Some callers want to add sysctls for global zones that + * may not yet exist so they pass a pointer to a pointer. + */ + if (arg2 == 0) + zone = *(uma_zone_t *)arg1; + else + zone = arg1; cur = uma_zone_get_cur(zone); return (sysctl_handle_int(oidp, &cur, 0, req)); } +static int +sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS) +{ + uma_zone_t zone = arg1; + uint64_t cur; + + cur = uma_zone_get_allocs(zone); + return (sysctl_handle_64(oidp, &cur, 0, req)); +} + +static int +sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS) +{ + uma_zone_t zone = arg1; + uint64_t cur; + + cur = uma_zone_get_frees(zone); + return (sysctl_handle_64(oidp, &cur, 0, req)); +} + #ifdef INVARIANTS static uma_slab_t uma_dbg_getslab(uma_zone_t zone, void *item) @@ -4500,7 +4681,8 @@ DB_SHOW_COMMAND(uma, db_show_uma) db_printf(fmt_entry, cur_zone->uz_name, (uintmax_t)cur_keg->uk_size, (intmax_t)used, cachefree, (uintmax_t)allocs, (uintmax_t)sleeps, - (unsigned)cur_zone->uz_count, (intmax_t)size, xdomain); + (unsigned)cur_zone->uz_bucket_size, (intmax_t)size, + xdomain); if (db_pager_quit) return; @@ -4525,7 +4707,7 @@ DB_SHOW_COMMAND(umacache, db_show_umacache) db_printf("%18s %8ju %8jd %8ld %12ju %8u\n", z->uz_name, (uintmax_t)z->uz_size, (intmax_t)(allocs - frees), cachefree, - (uintmax_t)allocs, z->uz_count); + (uintmax_t)allocs, z->uz_bucket_size); if (db_pager_quit) return; } Modified: head/sys/vm/uma_int.h ============================================================================== --- head/sys/vm/uma_int.h Wed Nov 27 23:19:06 2019 (r355148) +++ head/sys/vm/uma_int.h Thu Nov 28 00:19:09 2019 (r355149) @@ -338,8 +338,8 @@ struct uma_zone { uint64_t uz_items; /* Total items count */ uint64_t uz_max_items; /* Maximum number of items to alloc */ uint32_t uz_sleepers; /* Number of sleepers on memory */ - uint16_t uz_count; /* Amount of items in full bucket */ - uint16_t uz_count_max; /* Maximum amount of items there */ + uint16_t uz_bucket_size; /* Number of items in full bucket */ + uint16_t uz_bucket_size_max; /* Maximum number of bucket items */ /* Offset 64, used in bucket replenish. */ uma_import uz_import; /* Import new memory to cache. */ @@ -364,14 +364,17 @@ struct uma_zone { const char *uz_warning; /* Warning to print on failure */ struct timeval uz_ratecheck; /* Warnings rate-limiting */ struct task uz_maxaction; /* Task to run when at limit */ - uint16_t uz_count_min; /* Minimal amount of items in bucket */ + uint16_t uz_bucket_size_min; /* Min number of items in bucket */ - /* Offset 256, stats. */ + /* Offset 256+, stats and misc. */ counter_u64_t uz_allocs; /* Total number of allocations */ counter_u64_t uz_frees; /* Total number of frees */ counter_u64_t uz_fails; /* Total number of alloc failures */ uint64_t uz_sleeps; /* Total number of alloc sleeps */ uint64_t uz_xdomain; /* Total number of cross-domain frees */ + char *uz_ctlname; /* sysctl safe name string. */ + struct sysctl_oid *uz_oid; /* sysctl oid pointer. */ + int uz_namecnt; /* duplicate name count. */ /* * This HAS to be the last item because we adjust the zone size _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
