Schedule rotation work each pmu->hrtimer_interval_ms ms. Default to CMT_DEFAULT_ROTATION_PERIOD period.
Signed-off-by: David Carrillo-Cisneros <[email protected]> --- arch/x86/events/intel/cmt.c | 98 ++++++++++++++++++++++++++++++++++++++++++++- arch/x86/events/intel/cmt.h | 2 + 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/intel/cmt.c b/arch/x86/events/intel/cmt.c index 05803a8..8bf6aa5 100644 --- a/arch/x86/events/intel/cmt.c +++ b/arch/x86/events/intel/cmt.c @@ -51,6 +51,13 @@ static size_t pkg_uflags_size; static struct pkg_data **cmt_pkgs_data; /* + * Time between execution of rotation logic. The frequency of execution does + * not affect the rate at which RMIDs are recycled. + * The rotation period is stored in pmu->hrtimer_interval_ms. + */ +#define CMT_DEFAULT_ROTATION_PERIOD 1200 /* ms */ + +/* * Rotation Service Level Objectives (SLO) for monrs with llc_occupancy * monitoring. Note that these are monr level SLOs, therefore all pmonrs in * the monr meet or exceed them. @@ -1306,6 +1313,79 @@ static void smp_call_rmid_read(void *data) static struct pmu intel_cmt_pmu; +/* Schedule rotation in one package. */ +static bool __intel_cmt_schedule_rotation_for_pkg(struct pkg_data *pkgd) +{ + unsigned long delay; + + if (pkgd->work_cpu >= nr_cpu_ids) + return false; + delay = msecs_to_jiffies(intel_cmt_pmu.hrtimer_interval_ms); + + return schedule_delayed_work_on(pkgd->work_cpu, + &pkgd->rotation_work, delay); +} + +static void intel_cmt_schedule_rotation(void) +{ + struct pkg_data *pkgd = NULL; + + rcu_read_lock(); + while ((pkgd = cmt_pkgs_data_next_rcu(pkgd))) + __intel_cmt_schedule_rotation_for_pkg(pkgd); + rcu_read_unlock(); +} + +/* + * Rotation for @pkgd is needed if its package has at least one online CPU and: + * - there is a non-root monr (such monr could request rmids in @pkgd at any + * time), or + * - there are dirty rmids in pkgd. + */ +static bool intel_cmt_need_rmid_rotation(struct pkg_data *pkgd) +{ + unsigned long flags; + bool do_rot; + + /* protected by cmt_mutex. */ + if (!list_empty(&monr_hrchy_root->children)) + return true; + + raw_spin_lock_irqsave(&pkgd->lock, flags); + do_rot = pkgd->nr_dep_pmonrs || pkgd->nr_dirty_rmids; + raw_spin_unlock_irqrestore(&pkgd->lock, flags); + + return do_rot; +} + +/* + * Rotation function, runs per-package. + */ +static void intel_cmt_rmid_rotation_work(struct work_struct *work) +{ + struct pkg_data *pkgd; + + pkgd = container_of(to_delayed_work(work), + struct pkg_data, rotation_work); + + /* If this pkg_data is in its way to be destroyed. */ + if (pkgd->work_cpu >= nr_cpu_ids) + return; + + mutex_lock(&pkgd->mutex); + + if (!intel_cmt_need_rmid_rotation(pkgd)) + goto exit; + + /* To add call to rotation function in next patch */ + + if (intel_cmt_need_rmid_rotation(pkgd)) + __intel_cmt_schedule_rotation_for_pkg(pkgd); + +exit: + mutex_unlock(&pkgd->mutex); +} + /* Try to find a monr with same target, otherwise create new one. */ static int mon_group_setup_event(struct perf_event *event) { @@ -1796,6 +1876,11 @@ static int intel_cmt_event_init(struct perf_event *event) mutex_lock(&cmt_mutex); err = mon_group_setup_event(event); + /* + * schedule rotation even if error, in case the error was caused by + * insufficient rmids. + */ + intel_cmt_schedule_rotation(); mutex_unlock(&cmt_mutex); @@ -1885,6 +1970,7 @@ static const struct attribute_group *intel_cmt_attr_groups[] = { }; static struct pmu intel_cmt_pmu = { + .hrtimer_interval_ms = CMT_DEFAULT_ROTATION_PERIOD, .attr_groups = intel_cmt_attr_groups, .task_ctx_nr = perf_sw_context, .event_init = intel_cmt_event_init, @@ -2009,6 +2095,7 @@ static struct pkg_data *alloc_pkg_data(int cpu) mutex_init(&pkgd->mutex); raw_spin_lock_init(&pkgd->lock); + INIT_DELAYED_WORK(&pkgd->rotation_work, intel_cmt_rmid_rotation_work); pkgd->work_cpu = cpu; pkgd->pkgid = pkgid; @@ -2131,9 +2218,10 @@ static int intel_cmt_hp_online_enter(unsigned int cpu) rcu_read_lock(); pkgd = rcu_dereference(cmt_pkgs_data[pkgid]); - if (pkgd->work_cpu >= nr_cpu_ids) + if (pkgd->work_cpu >= nr_cpu_ids) { pkgd->work_cpu = cpu; - + __intel_cmt_schedule_rotation_for_pkg(pkgd); + } rcu_read_unlock(); return 0; @@ -2184,6 +2272,7 @@ static int intel_cmt_prep_down(unsigned int cpu) pkgd = rcu_dereference_protected(cmt_pkgs_data[pkgid], lockdep_is_held(&cmt_mutex)); if (pkgd->work_cpu >= nr_cpu_ids) { + cancel_delayed_work_sync(&pkgd->rotation_work); /* will destroy pkgd */ __terminate_pkg_data(pkgd); RCU_INIT_POINTER(cmt_pkgs_data[pkgid], NULL); @@ -2569,6 +2658,11 @@ static ssize_t cmt_monitoring_write(struct kernfs_open_file *of, monr_destroy(monr); exit_free: + /* + * schedule rotation even if in error, in case the error was caused by + * insufficient rmids. + */ + intel_cmt_schedule_rotation(); kfree(uflags); exit_unlock: monr_hrchy_release_mutexes(); diff --git a/arch/x86/events/intel/cmt.h b/arch/x86/events/intel/cmt.h index 8756666..872cce0 100644 --- a/arch/x86/events/intel/cmt.h +++ b/arch/x86/events/intel/cmt.h @@ -248,6 +248,7 @@ struct cmt_csd { * @mutex: Hold when modifying this pkg_data. * @mutex_key: lockdep class for pkg_data's mutex. * @lock: Hold to protect pmonrs in this pkg_data. + * @rotation_work: Task that performs rotation of rmids. * @work_cpu: CPU to run rotation and other batch jobs. * It must be in the package associated to its * instance of pkg_data. @@ -268,6 +269,7 @@ struct pkg_data { struct mutex mutex; raw_spinlock_t lock; + struct delayed_work rotation_work; unsigned int work_cpu; u32 max_rmid; u16 pkgid; -- 2.8.0.rc3.226.g39d4020
