Module Name: src
Committed By: maxv
Date: Sat Aug 17 12:37:49 UTC 2019
Modified Files:
src/sys/kern: subr_pool.c
Log Message:
Kernel Heap Hardening: use bitmaps on all off-page pools. This migrates 29
MI pools on amd64 from linked lists to bitmaps, which have higher security
properties.
Then, change the computation of the size of the PH pools: take into account
the bitmap area available by default in the ph_u2 union, and don't go with
&phpool[>0] if &phpool[0] already has enough space to embed a bitmap.
The pools that are migrated in this change all use bitmaps small enough to
fit in &phpool[0], therefore there is no increase in memory consumption.
To generate a diff of this commit:
cvs rdiff -u -r1.255 -r1.256 src/sys/kern/subr_pool.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/subr_pool.c
diff -u src/sys/kern/subr_pool.c:1.255 src/sys/kern/subr_pool.c:1.256
--- src/sys/kern/subr_pool.c:1.255 Fri Aug 16 10:41:35 2019
+++ src/sys/kern/subr_pool.c Sat Aug 17 12:37:49 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: subr_pool.c,v 1.255 2019/08/16 10:41:35 maxv Exp $ */
+/* $NetBSD: subr_pool.c,v 1.256 2019/08/17 12:37:49 maxv Exp $ */
/*
* Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015, 2018
@@ -33,7 +33,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.255 2019/08/16 10:41:35 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.256 2019/08/17 12:37:49 maxv Exp $");
#ifdef _KERNEL_OPT
#include "opt_ddb.h"
@@ -81,7 +81,7 @@ TAILQ_HEAD(, pool) pool_head = TAILQ_HEA
#define PHPOOL_MAX 8
static struct pool phpool[PHPOOL_MAX];
#define PHPOOL_FREELIST_NELEM(idx) \
- (((idx) == 0) ? 0 : BITMAP_SIZE * (1 << (idx)))
+ (((idx) == 0) ? BITMAP_MIN_SIZE : BITMAP_SIZE * (1 << (idx)))
#if defined(KASAN)
#define POOL_REDZONE
@@ -162,6 +162,7 @@ static unsigned int poolid_counter = 0;
typedef uint32_t pool_item_bitmap_t;
#define BITMAP_SIZE (CHAR_BIT * sizeof(pool_item_bitmap_t))
#define BITMAP_MASK (BITMAP_SIZE - 1)
+#define BITMAP_MIN_SIZE (CHAR_BIT * sizeof(((struct pool_item_header *)NULL)->ph_u2))
struct pool_item_header {
/* Page headers */
@@ -201,6 +202,9 @@ struct pool_item_header {
#define PHSIZE ALIGN(sizeof(struct pool_item_header))
+CTASSERT(offsetof(struct pool_item_header, ph_u2) +
+ BITMAP_MIN_SIZE / CHAR_BIT == sizeof(struct pool_item_header));
+
#if defined(DIAGNOSTIC) && !defined(KASAN)
#define POOL_CHECK_MAGIC
#endif
@@ -588,13 +592,11 @@ pool_subsystem_init(void)
size_t sz;
nelem = PHPOOL_FREELIST_NELEM(idx);
+ KASSERT(nelem != 0);
snprintf(phpool_names[idx], sizeof(phpool_names[idx]),
"phpool-%d", nelem);
- sz = sizeof(struct pool_item_header);
- if (nelem) {
- sz = offsetof(struct pool_item_header,
- ph_bitmap[howmany(nelem, BITMAP_SIZE)]);
- }
+ sz = offsetof(struct pool_item_header,
+ ph_bitmap[howmany(nelem, BITMAP_SIZE)]);
pool_init(&phpool[idx], sz, 0, 0, 0,
phpool_names[idx], &pool_allocator_meta, IPL_VM);
}
@@ -657,12 +659,16 @@ pool_init_is_usebmap(const struct pool *
}
/*
- * If we're on-page, and the page header can already contain a bitmap
- * big enough to cover all the items of the page, go with a bitmap.
+ * If we're off-page, go with a bitmap.
*/
if (!(pp->pr_roflags & PR_PHINPAGE)) {
- return false;
+ return true;
}
+
+ /*
+ * If we're on-page, and the page header can already contain a bitmap
+ * big enough to cover all the items of the page, go with a bitmap.
+ */
bmapsize = roundup(PHSIZE, pp->pr_align) -
offsetof(struct pool_item_header, ph_bitmap[0]);
KASSERT(bmapsize % sizeof(pool_item_bitmap_t) == 0);
@@ -801,14 +807,15 @@ pool_init(struct pool *pp, size_t size,
}
/*
- * If we're off-page and use a bitmap, choose the appropriate pool to
- * allocate page headers, whose size varies depending on the bitmap. If
- * we're just off-page, take the first pool, no extra size. If we're
- * on-page, nothing to do.
+ * If we're off-page, then we're using a bitmap; choose the appropriate
+ * pool to allocate page headers, whose size varies depending on the
+ * bitmap. If we're on-page, nothing to do.
*/
- if (!(pp->pr_roflags & PR_PHINPAGE) && (pp->pr_roflags & PR_USEBMAP)) {
+ if (!(pp->pr_roflags & PR_PHINPAGE)) {
int idx;
+ KASSERT(pp->pr_roflags & PR_USEBMAP);
+
for (idx = 0; pp->pr_itemsperpage > PHPOOL_FREELIST_NELEM(idx);
idx++) {
/* nothing */
@@ -823,8 +830,6 @@ pool_init(struct pool *pp, size_t size,
pp->pr_wchan, pp->pr_itemsperpage);
}
pp->pr_phpool = &phpool[idx];
- } else if (!(pp->pr_roflags & PR_PHINPAGE)) {
- pp->pr_phpool = &phpool[0];
} else {
pp->pr_phpool = NULL;
}
