For easier access, less dereferences and more consisent value, store
the group power in update_sg_lb_stats() and use it thereafter. The
actual value in sched_group::sched_group_power::power can change
throughout the load-balance pass if we're unlucky.
Signed-off-by: Peter Zijlstra <pet...@infradead.org>
---
kernel/sched/fair.c | 49 ++++++++++++++++++++++++++-----------------------
1 file changed, 26 insertions(+), 23 deletions(-)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -4284,6 +4284,7 @@ struct sg_lb_stats {
unsigned long group_load; /* Total load over the CPUs of the group */
unsigned long sum_weighted_load; /* Weighted load of group's tasks */
unsigned long load_per_task;
+ unsigned long group_power;
unsigned int sum_nr_running; /* Nr tasks running in the group */
unsigned int group_capacity;
unsigned int idle_cpus;
@@ -4320,11 +4321,11 @@ static inline void init_sd_lb_stats(stru
* long unsigned int avg_load; // 40 8
* long unsigned int group_load; // 48 8
* ...
- * } busiest_stat; // 40 56
- * struct sg_lb_stats this_stat; // 96 56
+ * } busiest_stat; // 40 64
+ * struct sg_lb_stats this_stat; // 104 64
*
- * // size: 152, cachelines: 3, members: 7
- * // last cacheline: 24 bytes
+ * // size: 168, cachelines: 3, members: 7
+ * // last cacheline: 40 bytes
* };
*
* Skimp on the clearing to avoid duplicate work. We can avoid clearing
@@ -4574,7 +4575,8 @@ static inline void update_sg_lb_stats(st
update_group_power(env->sd, env->dst_cpu);
/* Adjust by relative CPU power of the group */
- sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / group->sgp->power;
+ sgs->group_power = group->sgp->power;
+ sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / sgs->group_power;
/*
* Consider the group unbalanced when the imbalance is larger
@@ -4593,7 +4595,7 @@ static inline void update_sg_lb_stats(st
sgs->group_imb = 1;
sgs->group_capacity =
- DIV_ROUND_CLOSEST(group->sgp->power, SCHED_POWER_SCALE);
+ DIV_ROUND_CLOSEST(sgs->group_power, SCHED_POWER_SCALE);
if (!sgs->group_capacity)
sgs->group_capacity = fix_small_capacity(env->sd, group);
@@ -4696,7 +4698,7 @@ static inline void update_sd_lb_stats(st
/* Now, start updating sd_lb_stats */
sds->total_load += sgs->group_load;
- sds->total_pwr += sg->sgp->power;
+ sds->total_pwr += sgs->group_power;
if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) {
sds->busiest = sg;
@@ -4744,8 +4746,9 @@ static int check_asym_packing(struct lb_
if (env->dst_cpu > busiest_cpu)
return 0;
- env->imbalance = DIV_ROUND_CLOSEST(sds->busiest_stat.avg_load *
- sds->busiest->sgp->power, SCHED_POWER_SCALE);
+ env->imbalance = DIV_ROUND_CLOSEST(
+ sds->busiest_stat.avg_load * sds->busiest_stat.group_power,
+ SCHED_POWER_SCALE);
return 1;
}
@@ -4775,7 +4778,7 @@ void fix_small_imbalance(struct lb_env *
scaled_busy_load_per_task =
(busiest->load_per_task * SCHED_POWER_SCALE) /
- sds->busiest->sgp->power;
+ busiest->group_power;
if (busiest->avg_load - this->avg_load + scaled_busy_load_per_task >=
(scaled_busy_load_per_task * imbn)) {
@@ -4789,32 +4792,32 @@ void fix_small_imbalance(struct lb_env *
* moving them.
*/
- pwr_now += sds->busiest->sgp->power *
+ pwr_now += busiest->group_power *
min(busiest->load_per_task, busiest->avg_load);
- pwr_now += sds->this->sgp->power *
+ pwr_now += this->group_power *
min(this->load_per_task, this->avg_load);
pwr_now /= SCHED_POWER_SCALE;
/* Amount of load we'd subtract */
tmp = (busiest->load_per_task * SCHED_POWER_SCALE) /
- sds->busiest->sgp->power;
+ busiest->group_power;
if (busiest->avg_load > tmp) {
- pwr_move += sds->busiest->sgp->power *
+ pwr_move += busiest->group_power *
min(busiest->load_per_task,
busiest->avg_load - tmp);
}
/* Amount of load we'd add */
- if (busiest->avg_load * sds->busiest->sgp->power <
+ if (busiest->avg_load * busiest->group_power <
busiest->load_per_task * SCHED_POWER_SCALE) {
- tmp = (busiest->avg_load * sds->busiest->sgp->power) /
- sds->this->sgp->power;
+ tmp = (busiest->avg_load * busiest->group_power) /
+ this->group_power;
} else {
tmp = (busiest->load_per_task * SCHED_POWER_SCALE) /
- sds->this->sgp->power;
+ this->group_power;
}
- pwr_move += sds->this->sgp->power *
- min(this->load_per_task, this->avg_load + tmp);
+ pwr_move += this->group_power *
+ min(this->load_per_task, this->avg_load + tmp);
pwr_move /= SCHED_POWER_SCALE;
/* Move if we gain throughput */
@@ -4859,7 +4862,7 @@ static inline void calculate_imbalance(s
(busiest->sum_nr_running - busiest->group_capacity);
load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_POWER_SCALE);
- load_above_capacity /= sds->busiest->sgp->power;
+ load_above_capacity /= busiest->group_power;
}
/*
@@ -4877,8 +4880,8 @@ static inline void calculate_imbalance(s
/* How much load to actually move to equalise the imbalance */
env->imbalance = min(
- max_pull * sds->busiest->sgp->power,
- (sds->avg_load - this->avg_load) * sds->this->sgp->power
+ max_pull * busiest->group_power,
+ (sds->avg_load - this->avg_load) * this->group_power
) / SCHED_POWER_SCALE;
/*
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
