-----BEGIN PGP SIGNED MESSAGE----- Hash: SHA256 Am Tue, 11 Feb 2020 20:06:33 +0000 (UTC) Mark Johnston <ma...@freebsd.org> schrieb:
> Author: markj > Date: Tue Feb 11 20:06:33 2020 > New Revision: 357776 > URL: https://svnweb.freebsd.org/changeset/base/357776 > > Log: > Reduce lock hold time in keg_drain(). > > Maintain a count of free slabs in the per-domain keg structure and use > that to clear the free slab list in constant time for most cases. This > helps minimize lock contention induced by reclamation, in preparation > for proactive trimming of excesses of free memory. > > Reviewed by: jeff, rlibby > Tested by: pho > Differential Revision: https://reviews.freebsd.org/D23532 > > Modified: > head/sys/vm/uma_core.c > head/sys/vm/uma_int.h > > Modified: head/sys/vm/uma_core.c > ============================================================================== > --- head/sys/vm/uma_core.c Tue Feb 11 20:02:20 2020 (r357775) > +++ head/sys/vm/uma_core.c Tue Feb 11 20:06:33 2020 (r357776) > @@ -1258,39 +1258,34 @@ keg_free_slab(uma_keg_t keg, uma_slab_t slab, int star > static void > keg_drain(uma_keg_t keg) > { > - struct slabhead freeslabs = { 0 }; > + struct slabhead freeslabs; > uma_domain_t dom; > uma_slab_t slab, tmp; > int i, n; > > - /* > - * We don't want to take pages from statically allocated kegs at this > - * time > - */ > if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) > return; > > for (i = 0; i < vm_ndomains; i++) { > CTR4(KTR_UMA, "keg_drain %s(%p) domain %d free items: %u", > - keg->uk_name, keg, i, dom->ud_free); > - n = 0; > + keg->uk_name, keg, i, dom->ud_free_items); > dom = &keg->uk_domain[i]; > + LIST_INIT(&freeslabs); > + > KEG_LOCK(keg, i); > - LIST_FOREACH_SAFE(slab, &dom->ud_free_slab, us_link, tmp) { > - if (keg->uk_flags & UMA_ZFLAG_HASH) > + if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) { > + LIST_FOREACH(slab, &dom->ud_free_slab, us_link) > UMA_HASH_REMOVE(&keg->uk_hash, slab); > - n++; > - LIST_REMOVE(slab, us_link); > - LIST_INSERT_HEAD(&freeslabs, slab, us_link); > } > + n = dom->ud_free_slabs; > + LIST_SWAP(&freeslabs, &dom->ud_free_slab, uma_slab, us_link); > + dom->ud_free_slabs = 0; > + dom->ud_free_items -= n * keg->uk_ipers; > dom->ud_pages -= n * keg->uk_ppera; > - dom->ud_free -= n * keg->uk_ipers; > KEG_UNLOCK(keg, i); > - } > > - while ((slab = LIST_FIRST(&freeslabs)) != NULL) { > - LIST_REMOVE(slab, us_link); > - keg_free_slab(keg, slab, keg->uk_ipers); > + LIST_FOREACH_SAFE(slab, &freeslabs, us_link, tmp) > + keg_free_slab(keg, slab, keg->uk_ipers); > } > } > > @@ -1458,7 +1453,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int dom > dom = &keg->uk_domain[domain]; > LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); > dom->ud_pages += keg->uk_ppera; > - dom->ud_free += keg->uk_ipers; > + dom->ud_free_items += keg->uk_ipers; > > return (slab); > > @@ -2286,7 +2281,7 @@ zone_alloc_sysctl(uma_zone_t zone, void *unused) > "pages", CTLFLAG_RD, &dom->ud_pages, 0, > "Total pages currently allocated from VM"); > SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, > - "free", CTLFLAG_RD, &dom->ud_free, 0, > + "free_items", CTLFLAG_RD, &dom->ud_free_items, 0, > "items free in the slab layer"); > } > } else > @@ -2572,7 +2567,7 @@ keg_dtor(void *arg, int size, void *udata) > keg = (uma_keg_t)arg; > free = pages = 0; > for (i = 0; i < vm_ndomains; i++) { > - free += keg->uk_domain[i].ud_free; > + free += keg->uk_domain[i].ud_free_items; > pages += keg->uk_domain[i].ud_pages; > KEG_LOCK_FINI(keg, i); > } > @@ -3386,11 +3381,11 @@ keg_first_slab(uma_keg_t keg, int domain, bool rr) > start = domain; > do { > dom = &keg->uk_domain[domain]; > - if (!LIST_EMPTY(&dom->ud_part_slab)) > - return (LIST_FIRST(&dom->ud_part_slab)); > - if (!LIST_EMPTY(&dom->ud_free_slab)) { > - slab = LIST_FIRST(&dom->ud_free_slab); > + if ((slab = LIST_FIRST(&dom->ud_part_slab)) != NULL) > + return (slab); > + if ((slab = LIST_FIRST(&dom->ud_free_slab)) != NULL) { > LIST_REMOVE(slab, us_link); > + dom->ud_free_slabs--; > LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); > return (slab); > } > @@ -3417,7 +3412,7 @@ keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr > > KEG_LOCK(keg, domain); > reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve; > - if (keg->uk_domain[domain].ud_free <= reserve || > + if (keg->uk_domain[domain].ud_free_items <= reserve || > (slab = keg_first_slab(keg, domain, rr)) == NULL) { > KEG_UNLOCK(keg, domain); > return (NULL); > @@ -3502,9 +3497,13 @@ slab_alloc_item(uma_keg_t keg, uma_slab_t slab) > BIT_CLR(keg->uk_ipers, freei, &slab->us_free); > item = slab_item(slab, keg, freei); > slab->us_freecount--; > - dom->ud_free--; > + dom->ud_free_items--; > > - /* Move this slab to the full list */ > + /* > + * Move this slab to the full list. It must be on the partial list, so > + * we do not need to update the free slab count. In particular, > + * keg_fetch_slab() always returns slabs on the partial list. > + */ > if (slab->us_freecount == 0) { > LIST_REMOVE(slab, us_link); > LIST_INSERT_HEAD(&dom->ud_full_slab, slab, us_link); > @@ -3538,7 +3537,7 @@ zone_import(void *arg, void **bucket, int max, int dom > dom = &keg->uk_domain[slab->us_domain]; > while (slab->us_freecount && i < max) { > bucket[i++] = slab_alloc_item(keg, slab); > - if (dom->ud_free <= keg->uk_reserve) > + if (dom->ud_free_items <= keg->uk_reserve) > break; > #ifdef NUMA > /* > @@ -4240,9 +4239,10 @@ slab_free_item(uma_zone_t zone, uma_slab_t slab, void > > /* Do we need to remove from any lists? */ > dom = &keg->uk_domain[slab->us_domain]; > - if (slab->us_freecount+1 == keg->uk_ipers) { > + if (slab->us_freecount + 1 == keg->uk_ipers) { > LIST_REMOVE(slab, us_link); > LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link); > + dom->ud_free_slabs++; > } else if (slab->us_freecount == 0) { > LIST_REMOVE(slab, us_link); > LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); > @@ -4254,7 +4254,7 @@ slab_free_item(uma_zone_t zone, uma_slab_t slab, void > slab->us_freecount++; > > /* Keg statistics. */ > - dom->ud_free++; > + dom->ud_free_items++; > } > > static void > @@ -4635,9 +4635,14 @@ uma_prealloc(uma_zone_t zone, int items) > aflags); > if (slab != NULL) { > dom = &keg->uk_domain[slab->us_domain]; > + /* > + * keg_alloc_slab() always returns a slab on the > + * partial list. > + */ > LIST_REMOVE(slab, us_link); > LIST_INSERT_HEAD(&dom->ud_free_slab, slab, > us_link); > + dom->ud_free_slabs++; > KEG_UNLOCK(keg, slab->us_domain); > break; > } > @@ -4915,7 +4920,7 @@ sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) > LIST_FOREACH(kz, &uma_kegs, uk_link) { > kfree = pages = 0; > for (i = 0; i < vm_ndomains; i++) { > - kfree += kz->uk_domain[i].ud_free; > + kfree += kz->uk_domain[i].ud_free_items; > pages += kz->uk_domain[i].ud_pages; > } > LIST_FOREACH(z, &kz->uk_zones, uz_link) { > @@ -5219,7 +5224,7 @@ get_uma_stats(uma_keg_t kz, uma_zone_t z, uint64_t *al > *cachefree += z->uz_domain[i].uzd_nitems; > if (!((z->uz_flags & UMA_ZONE_SECONDARY) && > (LIST_FIRST(&kz->uk_zones) != z))) > - *cachefree += kz->uk_domain[i].ud_free; > + *cachefree += kz->uk_domain[i].ud_free_items; > } > *used = *allocs - frees; > return (((int64_t)*used + *cachefree) * kz->uk_size); > > Modified: head/sys/vm/uma_int.h > ============================================================================== > --- head/sys/vm/uma_int.h Tue Feb 11 20:02:20 2020 (r357775) > +++ head/sys/vm/uma_int.h Tue Feb 11 20:06:33 2020 (r357776) > @@ -324,7 +324,8 @@ struct uma_domain { > struct slabhead ud_free_slab; /* completely unallocated slabs */ > struct slabhead ud_full_slab; /* fully allocated slabs */ > uint32_t ud_pages; /* Total page count */ > - uint32_t ud_free; /* Count of items free in slabs */ > + uint32_t ud_free_items; /* Count of items free in all slabs */ > + uint32_t ud_free_slabs; /* Count of free slabs */ > } __aligned(CACHE_LINE_SIZE); > > typedef struct uma_domain * uma_domain_t; > _______________________________________________ > svn-src-h...@freebsd.org mailing list > https://lists.freebsd.org/mailman/listinfo/svn-src-head > To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org" This commit breaks buildworld: [...] ===> lib/libcasper/services/cap_fileargs (all) - --- all_subdir_lib/libmemstat --- - --- memstat_uma.o --- /usr/src/lib/libmemstat/memstat_uma.c:479:22: error: no member named 'ud_free' in 'struct uma_domain' kegfree += ukd.ud_free; ~~~ ^ 1 error generated. - --- all_subdir_lib/libiconv_modules --- - --- all_subdir_lib/libiconv_modules/VIQR --- ===> lib/libiconv_modules/VIQR (all) - -- O. Hartmann Ich widerspreche der Nutzung oder Übermittlung meiner Daten für Werbezwecke oder für die Markt- oder Meinungsforschung (§ 28 Abs. 4 BDSG). -----BEGIN PGP SIGNATURE----- iHUEARYIAB0WIQSy8IBxAPDkqVBaTJ44N1ZZPba5RwUCXkMKvQAKCRA4N1ZZPba5 R8tOAP0ZiRf9EKhz/QUBNoUfXjlfjCd3WuXilQKFMqNbOstaVgD/SkxfBxLE+C/P m0jmz7Al5AGai9Lia1X6y15McuqoHgc= =PPFe -----END PGP SIGNATURE----- _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
