this is a quick and simple change to add some randomness to pool
item allocation patterns.
it basically pregenerates 64 * 8 coin flips to see which end of the
free list items and pool cache pages should go on.
can someone check if there's a performance impact? i dont want to
prematurely complicate this if it's not expensive.
Index: subr_pool.c
===================================================================
RCS file: /cvs/src/sys/kern/subr_pool.c,v
retrieving revision 1.222
diff -u -p -r1.222 subr_pool.c
--- subr_pool.c 6 Feb 2018 22:35:32 -0000 1.222
+++ subr_pool.c 7 Feb 2018 04:58:30 -0000
@@ -277,6 +277,14 @@ int pool_wait_gc = 8;
RBT_PROTOTYPE(phtree, pool_page_header, ph_node, phtree_compare);
+#define POOL_RBYTES 128 /* XXX magic */
+#define POOL_RBITS (POOL_RBYTES * 8)
+
+static unsigned int pool_rbytes[POOL_RBYTES / sizeof(unsigned int)];
+static unsigned int pool_rbit = 0;
+
+static int pool_getrbit(void);
+
static inline int
phtree_compare(const struct pool_page_header *a,
const struct pool_page_header *b)
@@ -460,6 +468,8 @@ pool_init(struct pool *pp, size_t size,
TAILQ_INIT(&pp->pr_requests);
if (phpool.pr_size == 0) {
+ arc4random_buf(pool_rbytes, sizeof(pool_rbytes));
+
pool_init(&phpool, sizeof(struct pool_page_header), 0,
IPL_HIGH, 0, "phpool", NULL);
@@ -851,7 +861,11 @@ pool_do_put(struct pool *pp, void *v)
#endif /* DIAGNOSTIC */
pi->pi_magic = POOL_IMAGIC(ph, pi);
- XSIMPLEQ_INSERT_HEAD(&ph->ph_items, pi, pi_list);
+ if (pool_getrbit())
+ XSIMPLEQ_INSERT_HEAD(&ph->ph_items, pi, pi_list);
+ else
+ XSIMPLEQ_INSERT_TAIL(&ph->ph_items, pi, pi_list);
+
#ifdef DIAGNOSTIC
if (POOL_PHPOISON(ph))
poison_mem(pi + 1, pp->pr_size - sizeof(*pi));
@@ -1840,7 +1854,10 @@ pool_cache_list_free(struct pool *pp, st
pp->pr_cache_tick = ticks;
pp->pr_cache_nitems += POOL_CACHE_ITEM_NITEMS(ci);
- TAILQ_INSERT_TAIL(&pp->pr_cache_lists, ci, ci_nextl);
+ if (pool_getrbit())
+ TAILQ_INSERT_HEAD(&pp->pr_cache_lists, ci, ci_nextl);
+ else
+ TAILQ_INSERT_TAIL(&pp->pr_cache_lists, ci, ci_nextl);
pc->pc_nlput++;
@@ -2296,3 +2313,25 @@ static const struct pool_lock_ops pool_l
pool_lock_rw_assert_unlocked,
pool_lock_rw_sleep,
};
+
+static int
+pool_getrbit(void)
+{
+ unsigned int rbit;
+ unsigned int word;
+ unsigned int bit;
+
+ rbit = atomic_inc_int_nv(&pool_rbit) % POOL_RBITS;
+ if (rbit == 0) {
+ /*
+ * the next, but concurrent reader will get stale bits, but
+ * that is hard to predict, ie, sort of random?
+ */
+ arc4random_buf(pool_rbytes, sizeof(pool_rbytes));
+ }
+
+ word = rbit / (sizeof(word) * 8);
+ bit = rbit % (sizeof(word) * 8);
+
+ return (ISSET(pool_rbytes[word], 1 << bit));
+}
