The housekeeping_update() function currently allows update to the HK_TYPE_DOMAIN cpumask only. As we are going to enable dynamic modification of the other housekeeping cpumasks, we need to extend it to support passing in the information about the HK cpumask(s) to be updated. In cases where some HK cpumasks happen to be the same, it will be more efficient to update multiple HK cpumasks in one single call instead of calling it multiple times. Extend housekeeping_update() to support that as well.
Also add the restriction that passed in isolated cpumask parameter of housekeeping_update() must include all the CPUs isolated at boot time. This is currently the case for cpuset anyway. Signed-off-by: Waiman Long <[email protected]> --- include/linux/sched/isolation.h | 2 +- kernel/cgroup/cpuset.c | 2 +- kernel/sched/isolation.c | 99 +++++++++++++++++++++++---------- 3 files changed, 71 insertions(+), 32 deletions(-) diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h index d1707f121e20..a17f16e0156e 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h @@ -51,7 +51,7 @@ extern const struct cpumask *housekeeping_cpumask(enum hk_type type); extern bool housekeeping_enabled(enum hk_type type); extern void housekeeping_affine(struct task_struct *t, enum hk_type type); extern bool housekeeping_test_cpu(int cpu, enum hk_type type); -extern int housekeeping_update(struct cpumask *isol_mask); +extern int housekeeping_update(struct cpumask *isol_mask, unsigned long flags); extern void __init housekeeping_init(void); #else diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 1335e437098e..a4eccb0ec0d1 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1354,7 +1354,7 @@ static void cpuset_update_sd_hk_unlock(void) */ mutex_unlock(&cpuset_mutex); cpus_read_unlock(); - WARN_ON_ONCE(housekeeping_update(isolated_hk_cpus)); + WARN_ON_ONCE(housekeeping_update(isolated_hk_cpus, BIT(HK_TYPE_DOMAIN))); mutex_unlock(&cpuset_top_mutex); } else { cpuset_full_unlock(); diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 9ec9ae510dc7..965d6f8fe344 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -120,48 +120,87 @@ bool housekeeping_test_cpu(int cpu, enum hk_type type) } EXPORT_SYMBOL_GPL(housekeeping_test_cpu); -int housekeeping_update(struct cpumask *isol_mask) -{ - struct cpumask *trial, *old = NULL; - int err; +/* HK type processing table */ +static struct { + int type; + int boot_type; +} hk_types[] = { + { HK_TYPE_DOMAIN, HK_TYPE_DOMAIN_BOOT }, + { HK_TYPE_MANAGED_IRQ, HK_TYPE_MANAGED_IRQ_BOOT }, + { HK_TYPE_KERNEL_NOISE, HK_TYPE_KERNEL_NOISE_BOOT } +}; - trial = kmalloc(cpumask_size(), GFP_KERNEL); - if (!trial) - return -ENOMEM; +#define HK_TYPE_CNT ARRAY_SIZE(hk_types) - cpumask_andnot(trial, housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT), isol_mask); - if (!cpumask_intersects(trial, cpu_online_mask)) { - kfree(trial); - return -EINVAL; +int housekeeping_update(struct cpumask *isol_mask, unsigned long flags) +{ + struct cpumask *trial[HK_TYPE_CNT]; + int i, err = 0; + + for (i = 0; i < HK_TYPE_CNT; i++) { + int type = hk_types[i].type; + int boot = hk_types[i].boot_type; + + trial[i] = NULL; + if (flags & BIT(type)) { + trial[i] = kmalloc(cpumask_size(), GFP_KERNEL); + if (!trial[i]) { + err = -ENOMEM; + goto out; + } + /* + * The new HK cpumask must be a subset of its boot + * cpumask. + */ + cpumask_andnot(trial[i], cpu_possible_mask, isol_mask); + if (!cpumask_intersects(trial[i], cpu_online_mask) || + !cpumask_subset(trial[i], housekeeping_cpumask(boot))) { + i++; + err = -EINVAL; + goto out; + } + } } if (!housekeeping.flags) static_branch_enable(&housekeeping_overridden); - if (housekeeping.flags & HK_FLAG_DOMAIN) - old = housekeeping_cpumask_dereference(HK_TYPE_DOMAIN); - else - WRITE_ONCE(housekeeping.flags, housekeeping.flags | HK_FLAG_DOMAIN); - rcu_assign_pointer(housekeeping.cpumasks[HK_TYPE_DOMAIN], trial); - - synchronize_rcu(); - - pci_probe_flush_workqueue(); - mem_cgroup_flush_workqueue(); - vmstat_flush_workqueue(); + for (i = 0; i < HK_TYPE_CNT; i++) { + int type = hk_types[i].type; + struct cpumask *old; - err = workqueue_unbound_housekeeping_update(housekeeping_cpumask(HK_TYPE_DOMAIN)); - WARN_ON_ONCE(err < 0); + if (!trial[i]) + continue; + old = NULL; + if (housekeeping.flags & BIT(type)) + old = housekeeping_cpumask_dereference(type); + rcu_assign_pointer(housekeeping.cpumasks[type], trial[i]); + trial[i] = old; + } - err = tmigr_isolated_exclude_cpumask(isol_mask); - WARN_ON_ONCE(err < 0); + if ((housekeeping.flags & flags) != flags) + WRITE_ONCE(housekeeping.flags, housekeeping.flags | flags); - err = kthreads_update_housekeeping(); - WARN_ON_ONCE(err < 0); + synchronize_rcu(); - kfree(old); + if (flags & HK_FLAG_DOMAIN) { + /* + * HK_TYPE_DOMAIN specific callbacks + */ + pci_probe_flush_workqueue(); + mem_cgroup_flush_workqueue(); + vmstat_flush_workqueue(); + + WARN_ON_ONCE(workqueue_unbound_housekeeping_update( + housekeeping_cpumask(HK_TYPE_DOMAIN)) < 0); + WARN_ON_ONCE(tmigr_isolated_exclude_cpumask(isol_mask) < 0); + WARN_ON_ONCE(kthreads_update_housekeeping() < 0); + } - return 0; +out: + while (--i >= 0) + kfree(trial[i]); + return err; } void __init housekeeping_init(void) -- 2.53.0
