Periodically updates the buddy of a CPU according to the current activity of the system. A CPU is its own buddy if it participates to the packing effort. Otherwise, it points to a CPU that participates to the packing effort.
Signed-off-by: Vincent Guittot <[email protected]> --- kernel/sched/fair.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 5 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 234ecdd..28f8ea7 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -174,11 +174,17 @@ void sched_init_granularity(void) #ifdef CONFIG_SMP +static unsigned long power_of(int cpu) +{ + return cpu_rq(cpu)->cpu_power; +} + /* * Save the id of the optimal CPU that should be used to pack small tasks * The value -1 is used when no buddy has been found */ DEFINE_PER_CPU(int, sd_pack_buddy); +DEFINE_PER_CPU(struct sched_domain *, sd_pack_domain); /* * Look for the best buddy CPU that can be used to pack small tasks @@ -237,6 +243,68 @@ void update_packing_domain(int cpu) } pr_debug("CPU%d packing on CPU%d\n", cpu, id); + per_cpu(sd_pack_domain, cpu) = sd; + per_cpu(sd_pack_buddy, cpu) = id; +} + +void update_packing_buddy(int cpu, int activity) +{ + struct sched_domain *sd = per_cpu(sd_pack_domain, cpu); + struct sched_group *sg, *pack, *tmp; + int id = cpu; + + if (!sd) + return; + + /* + * The sched_domain of a CPU points on the local sched_group + * and this CPU of this local group is a good candidate + */ + pack = sg = sd->groups; + + /* loop the sched groups to find the best one */ + for (tmp = sg->next; tmp != sg; tmp = tmp->next) { + if ((tmp->sgp->power * pack->group_weight) > + (pack->sgp->power_available * tmp->group_weight)) + continue; + + if (((tmp->sgp->power * pack->group_weight) == + (pack->sgp->power * tmp->group_weight)) + && (cpumask_first(sched_group_cpus(tmp)) >= id)) + continue; + + /* we have found a better group */ + pack = tmp; + + /* Take the 1st CPU of the new group */ + id = cpumask_first(sched_group_cpus(pack)); + } + + if ((cpu == id) || (activity <= power_of(id))) { + per_cpu(sd_pack_buddy, cpu) = id; + return; + } + + for (tmp = pack; activity > 0; tmp = tmp->next) { + if (tmp->sgp->power > activity) { + id = cpumask_first(sched_group_cpus(tmp)); + activity -= power_of(id); + if (cpu == id) + activity = 0; + while ((activity > 0) && (id < nr_cpu_ids)) { + id = cpumask_next(id, sched_group_cpus(tmp)); + activity -= power_of(id); + if (cpu == id) + activity = 0; + } + } else if (cpumask_test_cpu(cpu, sched_group_cpus(tmp))) { + id = cpu; + activity = 0; + } else { + activity -= tmp->sgp->power; + } + } + per_cpu(sd_pack_buddy, cpu) = id; } @@ -3014,11 +3082,6 @@ static unsigned long target_load(int cpu, int type) return max(rq->cpu_load[type-1], total); } -static unsigned long power_of(int cpu) -{ - return cpu_rq(cpu)->cpu_power; -} - static unsigned long cpu_avg_load_per_task(int cpu) { struct rq *rq = cpu_rq(cpu); @@ -4740,6 +4803,22 @@ static bool update_sd_pick_busiest(struct lb_env *env, return false; } +static void update_plb_buddy(int cpu, int *balance, struct sd_lb_stats *sds, + struct sched_domain *sd) +{ + int buddy; + + if (sysctl_sched_packing_mode != SCHED_PACKING_FULL) + return; + + /* Update my buddy */ + if (sd == per_cpu(sd_pack_domain, cpu)) + update_packing_buddy(cpu, sds->total_activity); + + /* Get my new buddy */ + buddy = per_cpu(sd_pack_buddy, cpu); +} + /** * update_sd_lb_stats - Update sched_domain's statistics for load balancing. * @env: The load balancing environment. @@ -4807,6 +4886,8 @@ static inline void update_sd_lb_stats(struct lb_env *env, sg = sg->next; } while (sg != env->sd->groups); + + update_plb_buddy(env->dst_cpu, balance, sds, env->sd); } /** -- 1.7.9.5 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
