Re: [patch V2 22/67] bus/arm-cci: Convert to hotplug statemachine

[Punit Agrawal]

Anna-Maria Gleixner  writes:

> From: Sebastian Andrzej Siewior 
>
> Install the callbacks via the state machine and let the core invoke
> the callbacks on the already online CPUs.
>
> Signed-off-by: Sebastian Andrzej Siewior 
> Cc: Andrzej Hajda 
> Cc: Linus Torvalds 
> Cc: Mark Rutland 
> Cc: Olof Johansson 
> Cc: Peter Zijlstra 
> Cc: Punit Agrawal 
> Cc: Suzuki K Poulose 
> Cc: Suzuki K. Poulose 
> Cc: Thomas Gleixner 
> Cc: Will Deacon 
> Cc: Pawel Moll 
> Cc: linux-kernel@vger.kernel.org
> Signed-off-by: Anna-Maria Gleixner 

We don't expect to have multiple CCIs in the system. I suspect this got
changed from v1 along with ccn. Even then the patch still does the right
thing.

Acked-by: Punit Agrawal 

Thanks.


[...]

Re: [patch V2 22/67] bus/arm-cci: Convert to hotplug statemachine

[Punit Agrawal]

Anna-Maria Gleixner  writes:

> From: Sebastian Andrzej Siewior 
>
> Install the callbacks via the state machine and let the core invoke
> the callbacks on the already online CPUs.
>
> Signed-off-by: Sebastian Andrzej Siewior 
> Cc: Andrzej Hajda 
> Cc: Linus Torvalds 
> Cc: Mark Rutland 
> Cc: Olof Johansson 
> Cc: Peter Zijlstra 
> Cc: Punit Agrawal 
> Cc: Suzuki K Poulose 
> Cc: Suzuki K. Poulose 
> Cc: Thomas Gleixner 
> Cc: Will Deacon 
> Cc: Pawel Moll 
> Cc: linux-kernel@vger.kernel.org
> Signed-off-by: Anna-Maria Gleixner 

We don't expect to have multiple CCIs in the system. I suspect this got
changed from v1 along with ccn. Even then the patch still does the right
thing.

Acked-by: Punit Agrawal 

Thanks.


[...]

[patch V2 22/67] bus/arm-cci: Convert to hotplug statemachine

[Anna-Maria Gleixner]

From: Sebastian Andrzej Siewior 

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior 
Cc: Andrzej Hajda 
Cc: Linus Torvalds 
Cc: Mark Rutland 
Cc: Olof Johansson 
Cc: Peter Zijlstra 
Cc: Punit Agrawal 
Cc: Suzuki K Poulose 
Cc: Suzuki K. Poulose 
Cc: Thomas Gleixner 
Cc: Will Deacon 
Cc: Pawel Moll 
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Anna-Maria Gleixner 
---
 drivers/bus/arm-cci.c  |   53 +++--
 include/linux/cpuhotplug.h |1 
 2 files changed, 24 insertions(+), 30 deletions(-)

--- a/drivers/bus/arm-cci.c
+++ b/drivers/bus/arm-cci.c
@@ -144,12 +144,15 @@ struct cci_pmu {
int num_cntrs;
atomic_t active_events;
struct mutex reserve_mutex;
-   struct notifier_block cpu_nb;
+   struct list_head entry;
cpumask_t cpus;
 };
 
 #define to_cci_pmu(c)  (container_of(c, struct cci_pmu, pmu))
 
+static DEFINE_MUTEX(cci_pmu_mutex);
+static LIST_HEAD(cci_pmu_list);
+
 enum cci_models {
 #ifdef CONFIG_ARM_CCI400_PMU
CCI400_R0,
@@ -1503,31 +1506,26 @@ static int cci_pmu_init(struct cci_pmu *
return perf_pmu_register(_pmu->pmu, name, -1);
 }
 
-static int cci_pmu_cpu_notifier(struct notifier_block *self,
-   unsigned long action, void *hcpu)
+static int cci_pmu_offline_cpu(unsigned int cpu)
 {
-   struct cci_pmu *cci_pmu = container_of(self,
-   struct cci_pmu, cpu_nb);
-   unsigned int cpu = (long)hcpu;
+   struct cci_pmu *cci_pmu;
unsigned int target;
 
-   switch (action & ~CPU_TASKS_FROZEN) {
-   case CPU_DOWN_PREPARE:
+   mutex_lock(_pmu_mutex);
+   list_for_each_entry(cci_pmu, _pmu_list, entry) {
if (!cpumask_test_and_clear_cpu(cpu, _pmu->cpus))
-   break;
+   continue;
target = cpumask_any_but(cpu_online_mask, cpu);
-   if (target >= nr_cpu_ids) // UP, last CPU
-   break;
+   if (target >= nr_cpu_ids)
+   continue;
/*
 * TODO: migrate context once core races on event->ctx have
 * been fixed.
 */
cpumask_set_cpu(target, _pmu->cpus);
-   default:
-   break;
}
-
-   return NOTIFY_OK;
+   mutex_unlock(_pmu_mutex);
+   return 0;
 }
 
 static struct cci_pmu_model cci_pmu_models[] = {
@@ -1766,24 +1764,13 @@ static int cci_pmu_probe(struct platform
atomic_set(_pmu->active_events, 0);
cpumask_set_cpu(smp_processor_id(), _pmu->cpus);
 
-   cci_pmu->cpu_nb = (struct notifier_block) {
-   .notifier_call  = cci_pmu_cpu_notifier,
-   /*
-* to migrate uncore events, our notifier should be executed
-* before perf core's notifier.
-*/
-   .priority   = CPU_PRI_PERF + 1,
-   };
-
-   ret = register_cpu_notifier(_pmu->cpu_nb);
+   ret = cci_pmu_init(cci_pmu, pdev);
if (ret)
return ret;
 
-   ret = cci_pmu_init(cci_pmu, pdev);
-   if (ret) {
-   unregister_cpu_notifier(_pmu->cpu_nb);
-   return ret;
-   }
+   mutex_lock(_pmu_mutex);
+   list_add(_pmu->entry, _pmu_list);
+   mutex_unlock(_pmu_mutex);
 
pr_info("ARM %s PMU driver probed", cci_pmu->model->name);
return 0;
@@ -1817,6 +1804,12 @@ static int __init cci_platform_init(void
 {
int ret;
 
+   ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_CCI_ONLINE,
+   "AP_PERF_ARM_CCI_ONLINE", NULL,
+   cci_pmu_offline_cpu);
+   if (ret)
+   return ret;
+
ret = platform_driver_register(_pmu_driver);
if (ret)
return ret;
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -45,6 +45,7 @@ enum cpuhp_state {
CPUHP_AP_PERF_X86_CSTATE_ONLINE,
CPUHP_AP_PERF_S390_CF_ONLINE,
CPUHP_AP_PERF_S390_SF_ONLINE,
+   CPUHP_AP_PERF_ARM_CCI_ONLINE,
CPUHP_AP_NOTIFY_ONLINE,
CPUHP_AP_ONLINE_DYN,
CPUHP_AP_ONLINE_DYN_END = CPUHP_AP_ONLINE_DYN + 30,

[patch V2 22/67] bus/arm-cci: Convert to hotplug statemachine

[Anna-Maria Gleixner]

From: Sebastian Andrzej Siewior 

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior 
Cc: Andrzej Hajda 
Cc: Linus Torvalds 
Cc: Mark Rutland 
Cc: Olof Johansson 
Cc: Peter Zijlstra 
Cc: Punit Agrawal 
Cc: Suzuki K Poulose 
Cc: Suzuki K. Poulose 
Cc: Thomas Gleixner 
Cc: Will Deacon 
Cc: Pawel Moll 
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Anna-Maria Gleixner 
---
 drivers/bus/arm-cci.c  |   53 +++--
 include/linux/cpuhotplug.h |1 
 2 files changed, 24 insertions(+), 30 deletions(-)

--- a/drivers/bus/arm-cci.c
+++ b/drivers/bus/arm-cci.c
@@ -144,12 +144,15 @@ struct cci_pmu {
int num_cntrs;
atomic_t active_events;
struct mutex reserve_mutex;
-   struct notifier_block cpu_nb;
+   struct list_head entry;
cpumask_t cpus;
 };
 
 #define to_cci_pmu(c)  (container_of(c, struct cci_pmu, pmu))
 
+static DEFINE_MUTEX(cci_pmu_mutex);
+static LIST_HEAD(cci_pmu_list);
+
 enum cci_models {
 #ifdef CONFIG_ARM_CCI400_PMU
CCI400_R0,
@@ -1503,31 +1506,26 @@ static int cci_pmu_init(struct cci_pmu *
return perf_pmu_register(_pmu->pmu, name, -1);
 }
 
-static int cci_pmu_cpu_notifier(struct notifier_block *self,
-   unsigned long action, void *hcpu)
+static int cci_pmu_offline_cpu(unsigned int cpu)
 {
-   struct cci_pmu *cci_pmu = container_of(self,
-   struct cci_pmu, cpu_nb);
-   unsigned int cpu = (long)hcpu;
+   struct cci_pmu *cci_pmu;
unsigned int target;
 
-   switch (action & ~CPU_TASKS_FROZEN) {
-   case CPU_DOWN_PREPARE:
+   mutex_lock(_pmu_mutex);
+   list_for_each_entry(cci_pmu, _pmu_list, entry) {
if (!cpumask_test_and_clear_cpu(cpu, _pmu->cpus))
-   break;
+   continue;
target = cpumask_any_but(cpu_online_mask, cpu);
-   if (target >= nr_cpu_ids) // UP, last CPU
-   break;
+   if (target >= nr_cpu_ids)
+   continue;
/*
 * TODO: migrate context once core races on event->ctx have
 * been fixed.
 */
cpumask_set_cpu(target, _pmu->cpus);
-   default:
-   break;
}
-
-   return NOTIFY_OK;
+   mutex_unlock(_pmu_mutex);
+   return 0;
 }
 
 static struct cci_pmu_model cci_pmu_models[] = {
@@ -1766,24 +1764,13 @@ static int cci_pmu_probe(struct platform
atomic_set(_pmu->active_events, 0);
cpumask_set_cpu(smp_processor_id(), _pmu->cpus);
 
-   cci_pmu->cpu_nb = (struct notifier_block) {
-   .notifier_call  = cci_pmu_cpu_notifier,
-   /*
-* to migrate uncore events, our notifier should be executed
-* before perf core's notifier.
-*/
-   .priority   = CPU_PRI_PERF + 1,
-   };
-
-   ret = register_cpu_notifier(_pmu->cpu_nb);
+   ret = cci_pmu_init(cci_pmu, pdev);
if (ret)
return ret;
 
-   ret = cci_pmu_init(cci_pmu, pdev);
-   if (ret) {
-   unregister_cpu_notifier(_pmu->cpu_nb);
-   return ret;
-   }
+   mutex_lock(_pmu_mutex);
+   list_add(_pmu->entry, _pmu_list);
+   mutex_unlock(_pmu_mutex);
 
pr_info("ARM %s PMU driver probed", cci_pmu->model->name);
return 0;
@@ -1817,6 +1804,12 @@ static int __init cci_platform_init(void
 {
int ret;
 
+   ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_CCI_ONLINE,
+   "AP_PERF_ARM_CCI_ONLINE", NULL,
+   cci_pmu_offline_cpu);
+   if (ret)
+   return ret;
+
ret = platform_driver_register(_pmu_driver);
if (ret)
return ret;
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -45,6 +45,7 @@ enum cpuhp_state {
CPUHP_AP_PERF_X86_CSTATE_ONLINE,
CPUHP_AP_PERF_S390_CF_ONLINE,
CPUHP_AP_PERF_S390_SF_ONLINE,
+   CPUHP_AP_PERF_ARM_CCI_ONLINE,
CPUHP_AP_NOTIFY_ONLINE,
CPUHP_AP_ONLINE_DYN,
CPUHP_AP_ONLINE_DYN_END = CPUHP_AP_ONLINE_DYN + 30,