N_MEMORY_PRIVATE nodes hold device-managed memory that should not be used for general allocations. Without a gating mechanism, any allocation could land on a private node if it appears in the task's mems_allowed.
Introduce __GFP_PRIVATE that explicitly opts in to allocation from N_MEMORY_PRIVATE nodes. Add the GFP_PRIVATE compound mask (__GFP_PRIVATE | __GFP_THISNODE) for callers that explicitly target private nodes to help prevent fallback allocations from DRAM. Update cpuset_current_node_allowed() to filter out N_MEMORY_PRIVATE nodes unless __GFP_PRIVATE is set. In interrupt context, only N_MEMORY nodes are valid. Update cpuset_handle_hotplug() to include N_MEMORY_PRIVATE nodes in the effective mems set, allowing cgroup-level control over private node access. Signed-off-by: Gregory Price <[email protected]> --- include/linux/gfp_types.h | 15 +++++++++++++-- include/trace/events/mmflags.h | 4 ++-- kernel/cgroup/cpuset.c | 32 ++++++++++++++++++++++++++++---- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index 3de43b12209e..ac375f9a0fc2 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -33,7 +33,7 @@ enum { ___GFP_IO_BIT, ___GFP_FS_BIT, ___GFP_ZERO_BIT, - ___GFP_UNUSED_BIT, /* 0x200u unused */ + ___GFP_PRIVATE_BIT, ___GFP_DIRECT_RECLAIM_BIT, ___GFP_KSWAPD_RECLAIM_BIT, ___GFP_WRITE_BIT, @@ -69,7 +69,7 @@ enum { #define ___GFP_IO BIT(___GFP_IO_BIT) #define ___GFP_FS BIT(___GFP_FS_BIT) #define ___GFP_ZERO BIT(___GFP_ZERO_BIT) -/* 0x200u unused */ +#define ___GFP_PRIVATE BIT(___GFP_PRIVATE_BIT) #define ___GFP_DIRECT_RECLAIM BIT(___GFP_DIRECT_RECLAIM_BIT) #define ___GFP_KSWAPD_RECLAIM BIT(___GFP_KSWAPD_RECLAIM_BIT) #define ___GFP_WRITE BIT(___GFP_WRITE_BIT) @@ -139,6 +139,11 @@ enum { * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. * * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension. + * + * %__GFP_PRIVATE allows allocation from N_MEMORY_PRIVATE nodes (e.g., compressed + * memory, accelerator memory). Without this flag, allocations are restricted + * to N_MEMORY nodes only. Used by migration/demotion paths when explicitly + * targeting private nodes. */ #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) @@ -146,6 +151,7 @@ enum { #define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) #define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) #define __GFP_NO_OBJ_EXT ((__force gfp_t)___GFP_NO_OBJ_EXT) +#define __GFP_PRIVATE ((__force gfp_t)___GFP_PRIVATE) /** * DOC: Watermark modifiers @@ -367,6 +373,10 @@ enum { * available and will not wake kswapd/kcompactd on failure. The _LIGHT * version does not attempt reclaim/compaction at all and is by default used * in page fault path, while the non-light is used by khugepaged. + * + * %GFP_PRIVATE adds %__GFP_THISNODE by default to prevent any fallback + * allocations to other nodes, given that the caller was already attempting + * to access driver-managed memory explicitly. */ #define GFP_ATOMIC (__GFP_HIGH|__GFP_KSWAPD_RECLAIM) #define GFP_KERNEL (__GFP_RECLAIM | __GFP_IO | __GFP_FS) @@ -382,5 +392,6 @@ enum { #define GFP_TRANSHUGE_LIGHT ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \ __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM) #define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM) +#define GFP_PRIVATE (__GFP_PRIVATE | __GFP_THISNODE) #endif /* __LINUX_GFP_TYPES_H */ diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index a6e5a44c9b42..f042cd848451 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -37,7 +37,8 @@ TRACE_GFP_EM(HARDWALL) \ TRACE_GFP_EM(THISNODE) \ TRACE_GFP_EM(ACCOUNT) \ - TRACE_GFP_EM(ZEROTAGS) + TRACE_GFP_EM(ZEROTAGS) \ + TRACE_GFP_EM(PRIVATE) #ifdef CONFIG_KASAN_HW_TAGS # define TRACE_GFP_FLAGS_KASAN \ @@ -73,7 +74,6 @@ TRACE_GFP_FLAGS /* Just in case these are ever used */ -TRACE_DEFINE_ENUM(___GFP_UNUSED_BIT); TRACE_DEFINE_ENUM(___GFP_LAST_BIT); #define gfpflag_string(flag) {(__force unsigned long)flag, #flag} diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 473aa9261e16..1a597f0c7c6c 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -444,21 +444,32 @@ static void guarantee_active_cpus(struct task_struct *tsk, } /* - * Return in *pmask the portion of a cpusets's mems_allowed that + * Return in *pmask the portion of a cpuset's mems_allowed that * are online, with memory. If none are online with memory, walk * up the cpuset hierarchy until we find one that does have some * online mems. The top cpuset always has some mems online. * * One way or another, we guarantee to return some non-empty subset - * of node_states[N_MEMORY]. + * of node_states[N_MEMORY]. N_MEMORY_PRIVATE nodes from the + * original cpuset are preserved, but only N_MEMORY nodes are + * pulled from ancestors. * * Call with callback_lock or cpuset_mutex held. */ static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask) { + struct cpuset *orig_cs = cs; + int nid; + while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY])) cs = parent_cs(cs); + nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]); + + for_each_node_state(nid, N_MEMORY_PRIVATE) { + if (node_isset(nid, orig_cs->effective_mems)) + node_set(nid, *pmask); + } } /** @@ -4075,7 +4086,9 @@ static void cpuset_handle_hotplug(void) /* fetch the available cpus/mems and find out which changed how */ cpumask_copy(&new_cpus, cpu_active_mask); - new_mems = node_states[N_MEMORY]; + + /* Include N_MEMORY_PRIVATE so cpuset controls access the same way */ + nodes_or(new_mems, node_states[N_MEMORY], node_states[N_MEMORY_PRIVATE]); /* * If subpartitions_cpus is populated, it is likely that the check @@ -4488,10 +4501,21 @@ bool cpuset_node_allowed(struct cgroup *cgroup, int nid) * __alloc_pages() will include all nodes. If the slab allocator * is passed an offline node, it will fall back to the local node. * See kmem_cache_alloc_node(). + * + * + * Private nodes aren't eligible for these allocations, so skip them. + * guarantee_online_mems guaranttes at least one N_MEMORY node is set. */ static int cpuset_spread_node(int *rotor) { - return *rotor = next_node_in(*rotor, current->mems_allowed); + int node; + + do { + node = next_node_in(*rotor, current->mems_allowed); + *rotor = node; + } while (node_state(node, N_MEMORY_PRIVATE)); + + return node; } /** -- 2.53.0
