Re: [RFC PATCH v0 1/1] target/ppc: Support for H_RPT_INVALIDATE hcall
On Tue, 19 Jan 2021 10:14:55 +0530
Bharata B Rao wrote:
> On Fri, Jan 15, 2021 at 06:30:05PM +0100, Greg Kurz wrote:
> > On Fri, 15 Jan 2021 14:01:28 +0530
> > Bharata B Rao wrote:
> >
> > > On Wed, Jan 13, 2021 at 05:22:56PM +0100, Greg Kurz wrote:
> > > > Hi Bharata,
> > > >
> > > > On Wed, 6 Jan 2021 14:29:10 +0530
> > > > Bharata B Rao wrote:
> > > >
> > > > > If KVM_CAP_RPT_INVALIDATE KVM capability is enabled, then
> > > > >
> > > > > - indicate the availability of H_RPT_INVALIDATE hcall to the guest via
> > > > > ibm,hypertas-functions property.
> > > > > - Enable the hcall
> > > > >
> > > > > Both the above are done only if the new sPAPR machine capability
> > > > > cap-rpt-invalidate is set.
> > > > >
> > > > > Note: The KVM implementation of the hcall has been posted for upstream
> > > > > review here:
> > > > > https://lore.kernel.org/linuxppc-dev/20210105090557.2150104-1-bhar...@linux.ibm.com/T/#t
> > > > >
> > > > > Update to linux-headers/linux/kvm.h here is temporary, will be
> > > > > done via header updates once the kernel change is accepted upstream.
> > > > >
> > > > > Signed-off-by: Bharata B Rao
> > > > > ---
> > > >
> > > > Patch looks mostly fine. A few remarks below.
> > > >
> > > > > hw/ppc/spapr.c| 7 ++
> > > > > hw/ppc/spapr_caps.c | 49
> > > > > +++
> > > > > include/hw/ppc/spapr.h| 8 +--
> > > > > linux-headers/linux/kvm.h | 1 +
> > > > > target/ppc/kvm.c | 12 ++
> > > > > target/ppc/kvm_ppc.h | 11 +
> > > > > 6 files changed, 86 insertions(+), 2 deletions(-)
> > > > >
> > > > > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> > > > > index 489cefcb81..0228083800 100644
> > > > > --- a/hw/ppc/spapr.c
> > > > > +++ b/hw/ppc/spapr.c
> > > > > @@ -890,6 +890,11 @@ static void spapr_dt_rtas(SpaprMachineState
> > > > > *spapr, void *fdt)
> > > > > add_str(hypertas, "hcall-copy");
> > > > > add_str(hypertas, "hcall-debug");
> > > > > add_str(hypertas, "hcall-vphn");
> > > > > +if (kvm_enabled() &&
> > > >
> > > > You shouldn't check KVM here. The capability is enough to decide if we
> > > > should expose "hcall-rpt-invalidate" or not. FWIW we won't even reach
> > > > this code when running with anything but KVM.
> > >
> > > Correct, the capability itself can be only for KVM case.
> > >
> > > >
> > > > > +(spapr_get_cap(spapr, SPAPR_CAP_RPT_INVALIDATE) ==
> > > > > SPAPR_CAP_ON)) {
> > > > > +add_str(hypertas, "hcall-rpt-invalidate");
> > > > > +}
> > > > > +
> > > > > add_str(qemu_hypertas, "hcall-memop1");
> > > > >
> > > > > if (!kvm_enabled() || kvmppc_spapr_use_multitce()) {
> > > > > @@ -2021,6 +2026,7 @@ static const VMStateDescription vmstate_spapr =
> > > > > {
> > > > > _spapr_cap_ccf_assist,
> > > > > _spapr_cap_fwnmi,
> > > > > _spapr_fwnmi,
> > > > > +_spapr_cap_rpt_invalidate,
> > > > > NULL
> > > > > }
> > > > > };
> > > > > @@ -4478,6 +4484,7 @@ static void
> > > > > spapr_machine_class_init(ObjectClass *oc, void *data)
> > > > > smc->default_caps.caps[SPAPR_CAP_LARGE_DECREMENTER] =
> > > > > SPAPR_CAP_ON;
> > > > > smc->default_caps.caps[SPAPR_CAP_CCF_ASSIST] = SPAPR_CAP_ON;
> > > > > smc->default_caps.caps[SPAPR_CAP_FWNMI] = SPAPR_CAP_ON;
> > > > > +smc->default_caps.caps[SPAPR_CAP_RPT_INVALIDATE] = SPAPR_CAP_OFF;
> > > >
> > > > Any reason for not enabling this for the default machine type and
> > > > disabling it for existing machine types only ?
> > >
> > > If this capability is enabled, then
> > >
> > > 1. First level guest (L1) can off-load the TLB invalidations to the
> > > new hcall if the platform has disabled LPCR[GTSE].
> > >
> > > 2. Nested guest (L2) will switch to this new hcall rather than using
> > > the old H_TLB_INVALIDATE hcall.
> > >
> > > Case 2 is optional and case 1 makes sense only if LPCR[GTSE]=off.
> >
> > I don't think this is relevant, as the importance of each case can change,
> > e.g. nested is gaining momentum.
> >
> > > Hence I thought keeping it off by default and expecting the
> > > user to turn it on only if required would be correct.
> > >
> >
> > If the feature is an improvement, even for what is considered a corner
> > case now, and it doesn't do harm to setups that won't use it, then it
> > should be enabled IMHO.
> >
> > > Please note that turning this capability ON will result in the
> > > new hcall being exposed to the guest. I hope this is the right
> > > usage of spapr-caps?
> > >
> >
> > That's perfectly fine and this is why we should set it to ON
> > for the default machine type only.
>
> The property can be turned ON only when the hypervisor supports
> the hcall. So if it set to ON for default machine type, then
> it may fail if the host doesn't have this hcall. Hence I thought
> it should be OFF by default and turning ON should be left to the
> user.
>
Ok.
Re: [RFC PATCH v0 1/1] target/ppc: Support for H_RPT_INVALIDATE hcall
On Fri, Jan 15, 2021 at 06:30:05PM +0100, Greg Kurz wrote:
> On Fri, 15 Jan 2021 14:01:28 +0530
> Bharata B Rao wrote:
>
> > On Wed, Jan 13, 2021 at 05:22:56PM +0100, Greg Kurz wrote:
> > > Hi Bharata,
> > >
> > > On Wed, 6 Jan 2021 14:29:10 +0530
> > > Bharata B Rao wrote:
> > >
> > > > If KVM_CAP_RPT_INVALIDATE KVM capability is enabled, then
> > > >
> > > > - indicate the availability of H_RPT_INVALIDATE hcall to the guest via
> > > > ibm,hypertas-functions property.
> > > > - Enable the hcall
> > > >
> > > > Both the above are done only if the new sPAPR machine capability
> > > > cap-rpt-invalidate is set.
> > > >
> > > > Note: The KVM implementation of the hcall has been posted for upstream
> > > > review here:
> > > > https://lore.kernel.org/linuxppc-dev/20210105090557.2150104-1-bhar...@linux.ibm.com/T/#t
> > > >
> > > > Update to linux-headers/linux/kvm.h here is temporary, will be
> > > > done via header updates once the kernel change is accepted upstream.
> > > >
> > > > Signed-off-by: Bharata B Rao
> > > > ---
> > >
> > > Patch looks mostly fine. A few remarks below.
> > >
> > > > hw/ppc/spapr.c| 7 ++
> > > > hw/ppc/spapr_caps.c | 49 +++
> > > > include/hw/ppc/spapr.h| 8 +--
> > > > linux-headers/linux/kvm.h | 1 +
> > > > target/ppc/kvm.c | 12 ++
> > > > target/ppc/kvm_ppc.h | 11 +
> > > > 6 files changed, 86 insertions(+), 2 deletions(-)
> > > >
> > > > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> > > > index 489cefcb81..0228083800 100644
> > > > --- a/hw/ppc/spapr.c
> > > > +++ b/hw/ppc/spapr.c
> > > > @@ -890,6 +890,11 @@ static void spapr_dt_rtas(SpaprMachineState
> > > > *spapr, void *fdt)
> > > > add_str(hypertas, "hcall-copy");
> > > > add_str(hypertas, "hcall-debug");
> > > > add_str(hypertas, "hcall-vphn");
> > > > +if (kvm_enabled() &&
> > >
> > > You shouldn't check KVM here. The capability is enough to decide if we
> > > should expose "hcall-rpt-invalidate" or not. FWIW we won't even reach
> > > this code when running with anything but KVM.
> >
> > Correct, the capability itself can be only for KVM case.
> >
> > >
> > > > +(spapr_get_cap(spapr, SPAPR_CAP_RPT_INVALIDATE) ==
> > > > SPAPR_CAP_ON)) {
> > > > +add_str(hypertas, "hcall-rpt-invalidate");
> > > > +}
> > > > +
> > > > add_str(qemu_hypertas, "hcall-memop1");
> > > >
> > > > if (!kvm_enabled() || kvmppc_spapr_use_multitce()) {
> > > > @@ -2021,6 +2026,7 @@ static const VMStateDescription vmstate_spapr = {
> > > > _spapr_cap_ccf_assist,
> > > > _spapr_cap_fwnmi,
> > > > _spapr_fwnmi,
> > > > +_spapr_cap_rpt_invalidate,
> > > > NULL
> > > > }
> > > > };
> > > > @@ -4478,6 +4484,7 @@ static void spapr_machine_class_init(ObjectClass
> > > > *oc, void *data)
> > > > smc->default_caps.caps[SPAPR_CAP_LARGE_DECREMENTER] = SPAPR_CAP_ON;
> > > > smc->default_caps.caps[SPAPR_CAP_CCF_ASSIST] = SPAPR_CAP_ON;
> > > > smc->default_caps.caps[SPAPR_CAP_FWNMI] = SPAPR_CAP_ON;
> > > > +smc->default_caps.caps[SPAPR_CAP_RPT_INVALIDATE] = SPAPR_CAP_OFF;
> > >
> > > Any reason for not enabling this for the default machine type and
> > > disabling it for existing machine types only ?
> >
> > If this capability is enabled, then
> >
> > 1. First level guest (L1) can off-load the TLB invalidations to the
> > new hcall if the platform has disabled LPCR[GTSE].
> >
> > 2. Nested guest (L2) will switch to this new hcall rather than using
> > the old H_TLB_INVALIDATE hcall.
> >
> > Case 2 is optional and case 1 makes sense only if LPCR[GTSE]=off.
>
> I don't think this is relevant, as the importance of each case can change,
> e.g. nested is gaining momentum.
>
> > Hence I thought keeping it off by default and expecting the
> > user to turn it on only if required would be correct.
> >
>
> If the feature is an improvement, even for what is considered a corner
> case now, and it doesn't do harm to setups that won't use it, then it
> should be enabled IMHO.
>
> > Please note that turning this capability ON will result in the
> > new hcall being exposed to the guest. I hope this is the right
> > usage of spapr-caps?
> >
>
> That's perfectly fine and this is why we should set it to ON
> for the default machine type only.
The property can be turned ON only when the hypervisor supports
the hcall. So if it set to ON for default machine type, then
it may fail if the host doesn't have this hcall. Hence I thought
it should be OFF by default and turning ON should be left to the
user.
Regards,
Bharata.
Re: [RFC PATCH v0 1/1] target/ppc: Support for H_RPT_INVALIDATE hcall
On Fri, Jan 15, 2021 at 02:01:28PM +0530, Bharata B Rao wrote:
> On Wed, Jan 13, 2021 at 05:22:56PM +0100, Greg Kurz wrote:
> > Hi Bharata,
> >
> > On Wed, 6 Jan 2021 14:29:10 +0530
> > Bharata B Rao wrote:
> >
> > > If KVM_CAP_RPT_INVALIDATE KVM capability is enabled, then
> > >
> > > - indicate the availability of H_RPT_INVALIDATE hcall to the guest via
> > > ibm,hypertas-functions property.
> > > - Enable the hcall
> > >
> > > Both the above are done only if the new sPAPR machine capability
> > > cap-rpt-invalidate is set.
> > >
> > > Note: The KVM implementation of the hcall has been posted for upstream
> > > review here:
> > > https://lore.kernel.org/linuxppc-dev/20210105090557.2150104-1-bhar...@linux.ibm.com/T/#t
> > >
> > > Update to linux-headers/linux/kvm.h here is temporary, will be
> > > done via header updates once the kernel change is accepted upstream.
> > >
> > > Signed-off-by: Bharata B Rao
> > > ---
> >
> > Patch looks mostly fine. A few remarks below.
> >
> > > hw/ppc/spapr.c| 7 ++
> > > hw/ppc/spapr_caps.c | 49 +++
> > > include/hw/ppc/spapr.h| 8 +--
> > > linux-headers/linux/kvm.h | 1 +
> > > target/ppc/kvm.c | 12 ++
> > > target/ppc/kvm_ppc.h | 11 +
> > > 6 files changed, 86 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> > > index 489cefcb81..0228083800 100644
> > > --- a/hw/ppc/spapr.c
> > > +++ b/hw/ppc/spapr.c
> > > @@ -890,6 +890,11 @@ static void spapr_dt_rtas(SpaprMachineState *spapr,
> > > void *fdt)
> > > add_str(hypertas, "hcall-copy");
> > > add_str(hypertas, "hcall-debug");
> > > add_str(hypertas, "hcall-vphn");
> > > +if (kvm_enabled() &&
> >
> > You shouldn't check KVM here. The capability is enough to decide if we
> > should expose "hcall-rpt-invalidate" or not. FWIW we won't even reach
> > this code when running with anything but KVM.
>
> Correct, the capability itself can be only for KVM case.
Hrm.. that's kind of a problem in itself. Enabling KVM should not
change the guest visible environment.
>
> >
> > > +(spapr_get_cap(spapr, SPAPR_CAP_RPT_INVALIDATE) ==
> > > SPAPR_CAP_ON)) {
> > > +add_str(hypertas, "hcall-rpt-invalidate");
> > > +}
> > > +
> > > add_str(qemu_hypertas, "hcall-memop1");
> > >
> > > if (!kvm_enabled() || kvmppc_spapr_use_multitce()) {
> > > @@ -2021,6 +2026,7 @@ static const VMStateDescription vmstate_spapr = {
> > > _spapr_cap_ccf_assist,
> > > _spapr_cap_fwnmi,
> > > _spapr_fwnmi,
> > > +_spapr_cap_rpt_invalidate,
> > > NULL
> > > }
> > > };
> > > @@ -4478,6 +4484,7 @@ static void spapr_machine_class_init(ObjectClass
> > > *oc, void *data)
> > > smc->default_caps.caps[SPAPR_CAP_LARGE_DECREMENTER] = SPAPR_CAP_ON;
> > > smc->default_caps.caps[SPAPR_CAP_CCF_ASSIST] = SPAPR_CAP_ON;
> > > smc->default_caps.caps[SPAPR_CAP_FWNMI] = SPAPR_CAP_ON;
> > > +smc->default_caps.caps[SPAPR_CAP_RPT_INVALIDATE] = SPAPR_CAP_OFF;
> >
> > Any reason for not enabling this for the default machine type and
> > disabling it for existing machine types only ?
>
> If this capability is enabled, then
>
> 1. First level guest (L1) can off-load the TLB invalidations to the
> new hcall if the platform has disabled LPCR[GTSE].
>
> 2. Nested guest (L2) will switch to this new hcall rather than using
> the old H_TLB_INVALIDATE hcall.
>
> Case 2 is optional and case 1 makes sense only if LPCR[GTSE]=off.
> Hence I thought keeping it off by default and expecting the
> user to turn it on only if required would be correct.
>
> Please note that turning this capability ON will result in the
> new hcall being exposed to the guest. I hope this is the right
> usage of spapr-caps?
>
> > > diff --git a/target/ppc/kvm_ppc.h b/target/ppc/kvm_ppc.h
> > > index 73ce2bc951..8e27f8421f 100644
> > > --- a/target/ppc/kvm_ppc.h
> > > +++ b/target/ppc/kvm_ppc.h
> > > @@ -24,6 +24,7 @@ void kvmppc_enable_logical_ci_hcalls(void);
> > > void kvmppc_enable_set_mode_hcall(void);
> > > void kvmppc_enable_clear_ref_mod_hcalls(void);
> > > void kvmppc_enable_h_page_init(void);
> > > +void kvmppc_enable_h_rpt_invalidate(void);
> > > void kvmppc_set_papr(PowerPCCPU *cpu);
> > > int kvmppc_set_compat(PowerPCCPU *cpu, uint32_t compat_pvr);
> > > void kvmppc_set_mpic_proxy(PowerPCCPU *cpu, int mpic_proxy);
> > > @@ -72,6 +73,7 @@ bool kvmppc_has_cap_nested_kvm_hv(void);
> > > int kvmppc_set_cap_nested_kvm_hv(int enable);
> > > int kvmppc_get_cap_large_decr(void);
> > > int kvmppc_enable_cap_large_decr(PowerPCCPU *cpu, int enable);
> > > +int kvmppc_has_cap_rpt_invalidate(void);
> > > int kvmppc_enable_hwrng(void);
> > > int kvmppc_put_books_sregs(PowerPCCPU *cpu);
> > > PowerPCCPUClass *kvm_ppc_get_host_cpu_class(void);
> > > @@ -151,6 +153,10 @@ static inline void kvmppc_enable_h_page_init(void)
> > > {
> >
Re: [RFC PATCH v0 1/1] target/ppc: Support for H_RPT_INVALIDATE hcall
On Fri, 15 Jan 2021 14:01:28 +0530
Bharata B Rao wrote:
> On Wed, Jan 13, 2021 at 05:22:56PM +0100, Greg Kurz wrote:
> > Hi Bharata,
> >
> > On Wed, 6 Jan 2021 14:29:10 +0530
> > Bharata B Rao wrote:
> >
> > > If KVM_CAP_RPT_INVALIDATE KVM capability is enabled, then
> > >
> > > - indicate the availability of H_RPT_INVALIDATE hcall to the guest via
> > > ibm,hypertas-functions property.
> > > - Enable the hcall
> > >
> > > Both the above are done only if the new sPAPR machine capability
> > > cap-rpt-invalidate is set.
> > >
> > > Note: The KVM implementation of the hcall has been posted for upstream
> > > review here:
> > > https://lore.kernel.org/linuxppc-dev/20210105090557.2150104-1-bhar...@linux.ibm.com/T/#t
> > >
> > > Update to linux-headers/linux/kvm.h here is temporary, will be
> > > done via header updates once the kernel change is accepted upstream.
> > >
> > > Signed-off-by: Bharata B Rao
> > > ---
> >
> > Patch looks mostly fine. A few remarks below.
> >
> > > hw/ppc/spapr.c| 7 ++
> > > hw/ppc/spapr_caps.c | 49 +++
> > > include/hw/ppc/spapr.h| 8 +--
> > > linux-headers/linux/kvm.h | 1 +
> > > target/ppc/kvm.c | 12 ++
> > > target/ppc/kvm_ppc.h | 11 +
> > > 6 files changed, 86 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> > > index 489cefcb81..0228083800 100644
> > > --- a/hw/ppc/spapr.c
> > > +++ b/hw/ppc/spapr.c
> > > @@ -890,6 +890,11 @@ static void spapr_dt_rtas(SpaprMachineState *spapr,
> > > void *fdt)
> > > add_str(hypertas, "hcall-copy");
> > > add_str(hypertas, "hcall-debug");
> > > add_str(hypertas, "hcall-vphn");
> > > +if (kvm_enabled() &&
> >
> > You shouldn't check KVM here. The capability is enough to decide if we
> > should expose "hcall-rpt-invalidate" or not. FWIW we won't even reach
> > this code when running with anything but KVM.
>
> Correct, the capability itself can be only for KVM case.
>
> >
> > > +(spapr_get_cap(spapr, SPAPR_CAP_RPT_INVALIDATE) ==
> > > SPAPR_CAP_ON)) {
> > > +add_str(hypertas, "hcall-rpt-invalidate");
> > > +}
> > > +
> > > add_str(qemu_hypertas, "hcall-memop1");
> > >
> > > if (!kvm_enabled() || kvmppc_spapr_use_multitce()) {
> > > @@ -2021,6 +2026,7 @@ static const VMStateDescription vmstate_spapr = {
> > > _spapr_cap_ccf_assist,
> > > _spapr_cap_fwnmi,
> > > _spapr_fwnmi,
> > > +_spapr_cap_rpt_invalidate,
> > > NULL
> > > }
> > > };
> > > @@ -4478,6 +4484,7 @@ static void spapr_machine_class_init(ObjectClass
> > > *oc, void *data)
> > > smc->default_caps.caps[SPAPR_CAP_LARGE_DECREMENTER] = SPAPR_CAP_ON;
> > > smc->default_caps.caps[SPAPR_CAP_CCF_ASSIST] = SPAPR_CAP_ON;
> > > smc->default_caps.caps[SPAPR_CAP_FWNMI] = SPAPR_CAP_ON;
> > > +smc->default_caps.caps[SPAPR_CAP_RPT_INVALIDATE] = SPAPR_CAP_OFF;
> >
> > Any reason for not enabling this for the default machine type and
> > disabling it for existing machine types only ?
>
> If this capability is enabled, then
>
> 1. First level guest (L1) can off-load the TLB invalidations to the
> new hcall if the platform has disabled LPCR[GTSE].
>
> 2. Nested guest (L2) will switch to this new hcall rather than using
> the old H_TLB_INVALIDATE hcall.
>
> Case 2 is optional and case 1 makes sense only if LPCR[GTSE]=off.
I don't think this is relevant, as the importance of each case can change,
e.g. nested is gaining momentum.
> Hence I thought keeping it off by default and expecting the
> user to turn it on only if required would be correct.
>
If the feature is an improvement, even for what is considered a corner
case now, and it doesn't do harm to setups that won't use it, then it
should be enabled IMHO.
> Please note that turning this capability ON will result in the
> new hcall being exposed to the guest. I hope this is the right
> usage of spapr-caps?
>
That's perfectly fine and this is why we should set it to ON
for the default machine type only.
> > > diff --git a/target/ppc/kvm_ppc.h b/target/ppc/kvm_ppc.h
> > > index 73ce2bc951..8e27f8421f 100644
> > > --- a/target/ppc/kvm_ppc.h
> > > +++ b/target/ppc/kvm_ppc.h
> > > @@ -24,6 +24,7 @@ void kvmppc_enable_logical_ci_hcalls(void);
> > > void kvmppc_enable_set_mode_hcall(void);
> > > void kvmppc_enable_clear_ref_mod_hcalls(void);
> > > void kvmppc_enable_h_page_init(void);
> > > +void kvmppc_enable_h_rpt_invalidate(void);
> > > void kvmppc_set_papr(PowerPCCPU *cpu);
> > > int kvmppc_set_compat(PowerPCCPU *cpu, uint32_t compat_pvr);
> > > void kvmppc_set_mpic_proxy(PowerPCCPU *cpu, int mpic_proxy);
> > > @@ -72,6 +73,7 @@ bool kvmppc_has_cap_nested_kvm_hv(void);
> > > int kvmppc_set_cap_nested_kvm_hv(int enable);
> > > int kvmppc_get_cap_large_decr(void);
> > > int kvmppc_enable_cap_large_decr(PowerPCCPU *cpu, int enable);
> > > +int
Re: [RFC PATCH v0 1/1] target/ppc: Support for H_RPT_INVALIDATE hcall
On Wed, Jan 13, 2021 at 05:22:56PM +0100, Greg Kurz wrote:
> Hi Bharata,
>
> On Wed, 6 Jan 2021 14:29:10 +0530
> Bharata B Rao wrote:
>
> > If KVM_CAP_RPT_INVALIDATE KVM capability is enabled, then
> >
> > - indicate the availability of H_RPT_INVALIDATE hcall to the guest via
> > ibm,hypertas-functions property.
> > - Enable the hcall
> >
> > Both the above are done only if the new sPAPR machine capability
> > cap-rpt-invalidate is set.
> >
> > Note: The KVM implementation of the hcall has been posted for upstream
> > review here:
> > https://lore.kernel.org/linuxppc-dev/20210105090557.2150104-1-bhar...@linux.ibm.com/T/#t
> >
> > Update to linux-headers/linux/kvm.h here is temporary, will be
> > done via header updates once the kernel change is accepted upstream.
> >
> > Signed-off-by: Bharata B Rao
> > ---
>
> Patch looks mostly fine. A few remarks below.
>
> > hw/ppc/spapr.c| 7 ++
> > hw/ppc/spapr_caps.c | 49 +++
> > include/hw/ppc/spapr.h| 8 +--
> > linux-headers/linux/kvm.h | 1 +
> > target/ppc/kvm.c | 12 ++
> > target/ppc/kvm_ppc.h | 11 +
> > 6 files changed, 86 insertions(+), 2 deletions(-)
> >
> > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> > index 489cefcb81..0228083800 100644
> > --- a/hw/ppc/spapr.c
> > +++ b/hw/ppc/spapr.c
> > @@ -890,6 +890,11 @@ static void spapr_dt_rtas(SpaprMachineState *spapr,
> > void *fdt)
> > add_str(hypertas, "hcall-copy");
> > add_str(hypertas, "hcall-debug");
> > add_str(hypertas, "hcall-vphn");
> > +if (kvm_enabled() &&
>
> You shouldn't check KVM here. The capability is enough to decide if we
> should expose "hcall-rpt-invalidate" or not. FWIW we won't even reach
> this code when running with anything but KVM.
Correct, the capability itself can be only for KVM case.
>
> > +(spapr_get_cap(spapr, SPAPR_CAP_RPT_INVALIDATE) == SPAPR_CAP_ON)) {
> > +add_str(hypertas, "hcall-rpt-invalidate");
> > +}
> > +
> > add_str(qemu_hypertas, "hcall-memop1");
> >
> > if (!kvm_enabled() || kvmppc_spapr_use_multitce()) {
> > @@ -2021,6 +2026,7 @@ static const VMStateDescription vmstate_spapr = {
> > _spapr_cap_ccf_assist,
> > _spapr_cap_fwnmi,
> > _spapr_fwnmi,
> > +_spapr_cap_rpt_invalidate,
> > NULL
> > }
> > };
> > @@ -4478,6 +4484,7 @@ static void spapr_machine_class_init(ObjectClass *oc,
> > void *data)
> > smc->default_caps.caps[SPAPR_CAP_LARGE_DECREMENTER] = SPAPR_CAP_ON;
> > smc->default_caps.caps[SPAPR_CAP_CCF_ASSIST] = SPAPR_CAP_ON;
> > smc->default_caps.caps[SPAPR_CAP_FWNMI] = SPAPR_CAP_ON;
> > +smc->default_caps.caps[SPAPR_CAP_RPT_INVALIDATE] = SPAPR_CAP_OFF;
>
> Any reason for not enabling this for the default machine type and
> disabling it for existing machine types only ?
If this capability is enabled, then
1. First level guest (L1) can off-load the TLB invalidations to the
new hcall if the platform has disabled LPCR[GTSE].
2. Nested guest (L2) will switch to this new hcall rather than using
the old H_TLB_INVALIDATE hcall.
Case 2 is optional and case 1 makes sense only if LPCR[GTSE]=off.
Hence I thought keeping it off by default and expecting the
user to turn it on only if required would be correct.
Please note that turning this capability ON will result in the
new hcall being exposed to the guest. I hope this is the right
usage of spapr-caps?
> > diff --git a/target/ppc/kvm_ppc.h b/target/ppc/kvm_ppc.h
> > index 73ce2bc951..8e27f8421f 100644
> > --- a/target/ppc/kvm_ppc.h
> > +++ b/target/ppc/kvm_ppc.h
> > @@ -24,6 +24,7 @@ void kvmppc_enable_logical_ci_hcalls(void);
> > void kvmppc_enable_set_mode_hcall(void);
> > void kvmppc_enable_clear_ref_mod_hcalls(void);
> > void kvmppc_enable_h_page_init(void);
> > +void kvmppc_enable_h_rpt_invalidate(void);
> > void kvmppc_set_papr(PowerPCCPU *cpu);
> > int kvmppc_set_compat(PowerPCCPU *cpu, uint32_t compat_pvr);
> > void kvmppc_set_mpic_proxy(PowerPCCPU *cpu, int mpic_proxy);
> > @@ -72,6 +73,7 @@ bool kvmppc_has_cap_nested_kvm_hv(void);
> > int kvmppc_set_cap_nested_kvm_hv(int enable);
> > int kvmppc_get_cap_large_decr(void);
> > int kvmppc_enable_cap_large_decr(PowerPCCPU *cpu, int enable);
> > +int kvmppc_has_cap_rpt_invalidate(void);
> > int kvmppc_enable_hwrng(void);
> > int kvmppc_put_books_sregs(PowerPCCPU *cpu);
> > PowerPCCPUClass *kvm_ppc_get_host_cpu_class(void);
> > @@ -151,6 +153,10 @@ static inline void kvmppc_enable_h_page_init(void)
> > {
> > }
> >
> > +static inline void kvmppc_enable_h_rpt_invalidate(void)
> > +{
>
> g_assert_not_reached() ?
Don't see many others doing that, is that a new preferred
way?
Regards,
Bharata.
Re: [RFC PATCH v0 1/1] target/ppc: Support for H_RPT_INVALIDATE hcall
Hi Bharata,
On Wed, 6 Jan 2021 14:29:10 +0530
Bharata B Rao wrote:
> If KVM_CAP_RPT_INVALIDATE KVM capability is enabled, then
>
> - indicate the availability of H_RPT_INVALIDATE hcall to the guest via
> ibm,hypertas-functions property.
> - Enable the hcall
>
> Both the above are done only if the new sPAPR machine capability
> cap-rpt-invalidate is set.
>
> Note: The KVM implementation of the hcall has been posted for upstream
> review here:
> https://lore.kernel.org/linuxppc-dev/20210105090557.2150104-1-bhar...@linux.ibm.com/T/#t
>
> Update to linux-headers/linux/kvm.h here is temporary, will be
> done via header updates once the kernel change is accepted upstream.
>
> Signed-off-by: Bharata B Rao
> ---
Patch looks mostly fine. A few remarks below.
> hw/ppc/spapr.c| 7 ++
> hw/ppc/spapr_caps.c | 49 +++
> include/hw/ppc/spapr.h| 8 +--
> linux-headers/linux/kvm.h | 1 +
> target/ppc/kvm.c | 12 ++
> target/ppc/kvm_ppc.h | 11 +
> 6 files changed, 86 insertions(+), 2 deletions(-)
>
> diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> index 489cefcb81..0228083800 100644
> --- a/hw/ppc/spapr.c
> +++ b/hw/ppc/spapr.c
> @@ -890,6 +890,11 @@ static void spapr_dt_rtas(SpaprMachineState *spapr, void
> *fdt)
> add_str(hypertas, "hcall-copy");
> add_str(hypertas, "hcall-debug");
> add_str(hypertas, "hcall-vphn");
> +if (kvm_enabled() &&
You shouldn't check KVM here. The capability is enough to decide if we
should expose "hcall-rpt-invalidate" or not. FWIW we won't even reach
this code when running with anything but KVM.
> +(spapr_get_cap(spapr, SPAPR_CAP_RPT_INVALIDATE) == SPAPR_CAP_ON)) {
> +add_str(hypertas, "hcall-rpt-invalidate");
> +}
> +
> add_str(qemu_hypertas, "hcall-memop1");
>
> if (!kvm_enabled() || kvmppc_spapr_use_multitce()) {
> @@ -2021,6 +2026,7 @@ static const VMStateDescription vmstate_spapr = {
> _spapr_cap_ccf_assist,
> _spapr_cap_fwnmi,
> _spapr_fwnmi,
> +_spapr_cap_rpt_invalidate,
> NULL
> }
> };
> @@ -4478,6 +4484,7 @@ static void spapr_machine_class_init(ObjectClass *oc,
> void *data)
> smc->default_caps.caps[SPAPR_CAP_LARGE_DECREMENTER] = SPAPR_CAP_ON;
> smc->default_caps.caps[SPAPR_CAP_CCF_ASSIST] = SPAPR_CAP_ON;
> smc->default_caps.caps[SPAPR_CAP_FWNMI] = SPAPR_CAP_ON;
> +smc->default_caps.caps[SPAPR_CAP_RPT_INVALIDATE] = SPAPR_CAP_OFF;
Any reason for not enabling this for the default machine type and
disabling it for existing machine types only ?
> spapr_caps_add_properties(smc);
> smc->irq = _irq_dual;
> smc->dr_phb_enabled = true;
> diff --git a/hw/ppc/spapr_caps.c b/hw/ppc/spapr_caps.c
> index 9341e9782a..ebf81e3b23 100644
> --- a/hw/ppc/spapr_caps.c
> +++ b/hw/ppc/spapr_caps.c
> @@ -524,6 +524,45 @@ static void cap_fwnmi_apply(SpaprMachineState *spapr,
> uint8_t val,
> }
> }
>
> +static void cap_rpt_invalidate_apply(SpaprMachineState *spapr,
> + uint8_t val, Error **errp)
> +{
> +ERRP_GUARD();
> +PowerPCCPU *cpu = POWERPC_CPU(first_cpu);
> +
> +if (!val) {
> +/* capability disabled by default */
> +return;
> +}
> +
> +if (tcg_enabled()) {
> +error_setg(errp, "No H_RPT_INVALIDATE support in TCG");
> +error_append_hint(errp, "Try appending -machine
> cap-rpt-invalidate=off\n");
> +} else if (kvm_enabled()) {
> +if (!ppc_check_compat(cpu, CPU_POWERPC_LOGICAL_3_00, 0,
> + spapr->max_compat_pvr)) {
> +error_setg(errp, "H_RPT_INVALIDATE only supported on POWER9");
> +error_append_hint(errp,
> + "Try appending -machine
> max-cpu-compat=power9\n");
> +return;
> +}
> +
> +if (!kvmppc_has_cap_mmu_radix()) {
> +error_setg(errp, "H_RPT_INVALIDATE only supported on Radix");
> +return;
> +}
> +
> +if (!kvmppc_has_cap_rpt_invalidate()) {
> +error_setg(errp,
> + "KVM implementation does not support
> H_RPT_INVALIDATE");
> +error_append_hint(errp,
> + "Try appending -machine
> cap-rpt-invalidate=off\n");
> +} else {
> +kvmppc_enable_h_rpt_invalidate();
> +}
> +}
> +}
> +
> SpaprCapabilityInfo capability_table[SPAPR_CAP_NUM] = {
> [SPAPR_CAP_HTM] = {
> .name = "htm",
> @@ -632,6 +671,15 @@ SpaprCapabilityInfo capability_table[SPAPR_CAP_NUM] = {
> .type = "bool",
> .apply = cap_fwnmi_apply,
> },
> +[SPAPR_CAP_RPT_INVALIDATE] = {
> +.name = "rpt-invalidate",
> +.description = "Allow H_RPT_INVALIDATE",
> +.index = SPAPR_CAP_RPT_INVALIDATE,
> +.get = spapr_cap_get_bool,
> +.set = spapr_cap_set_bool,
> +
Re: [RFC PATCH v0 1/1] target/ppc: Support for H_RPT_INVALIDATE hcall
On Tue, Jan 12, 2021 at 10:16:30AM -0300, Daniel Henrique Barboza wrote: > > > On 1/6/21 5:59 AM, Bharata B Rao wrote: > > If KVM_CAP_RPT_INVALIDATE KVM capability is enabled, then > > > > - indicate the availability of H_RPT_INVALIDATE hcall to the guest via > >ibm,hypertas-functions property. > > - Enable the hcall > > > > Both the above are done only if the new sPAPR machine capability > > cap-rpt-invalidate is set. > > > > Note: The KVM implementation of the hcall has been posted for upstream > > review here: > > https://lore.kernel.org/linuxppc-dev/20210105090557.2150104-1-bhar...@linux.ibm.com/T/#t > > > > Update to linux-headers/linux/kvm.h here is temporary, will be > > done via header updates once the kernel change is accepted upstream. > > > > Signed-off-by: Bharata B Rao > > --- > > Code LGTM. > > Reviewed-by: Daniel Henrique Barboza > > > A few questions about the logic: > > - does it work only on Power 9 like you mentioned in the error message > down below? If it's supported on Power 10 as well then we would want the > error message to read "H_RPT_INVALIDATE only supported on POWER9 and newer" > to contemplate it. Making it conditional to Power 9 was an oversight, will remove in the next iteration. > > - Does it make sense to expose "rpt-invalidate" to Libvirt? I see that the > capability is turned off by default, which may indicate that even if kernel > and QEMU support is present the user might want to not enable it. Is there > some sort of drawback/compromise when activating this cap? I have added this to take care of migration compatibility between source and target when hcall is present in target and not present in source or vice versa. I wonder if there is any other preferred method than introducing a new machine capability like cap-rpt-invalidate. Regards, Bharata.
Re: [RFC PATCH v0 1/1] target/ppc: Support for H_RPT_INVALIDATE hcall
On 1/6/21 5:59 AM, Bharata B Rao wrote:
If KVM_CAP_RPT_INVALIDATE KVM capability is enabled, then
- indicate the availability of H_RPT_INVALIDATE hcall to the guest via
ibm,hypertas-functions property.
- Enable the hcall
Both the above are done only if the new sPAPR machine capability
cap-rpt-invalidate is set.
Note: The KVM implementation of the hcall has been posted for upstream
review here:
https://lore.kernel.org/linuxppc-dev/20210105090557.2150104-1-bhar...@linux.ibm.com/T/#t
Update to linux-headers/linux/kvm.h here is temporary, will be
done via header updates once the kernel change is accepted upstream.
Signed-off-by: Bharata B Rao
---
Code LGTM.
Reviewed-by: Daniel Henrique Barboza
A few questions about the logic:
- does it work only on Power 9 like you mentioned in the error message
down below? If it's supported on Power 10 as well then we would want the
error message to read "H_RPT_INVALIDATE only supported on POWER9 and newer"
to contemplate it.
- Does it make sense to expose "rpt-invalidate" to Libvirt? I see that the
capability is turned off by default, which may indicate that even if kernel
and QEMU support is present the user might want to not enable it. Is there
some sort of drawback/compromise when activating this cap?
Thanks,
DHB
hw/ppc/spapr.c| 7 ++
hw/ppc/spapr_caps.c | 49 +++
include/hw/ppc/spapr.h| 8 +--
linux-headers/linux/kvm.h | 1 +
target/ppc/kvm.c | 12 ++
target/ppc/kvm_ppc.h | 11 +
6 files changed, 86 insertions(+), 2 deletions(-)
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index 489cefcb81..0228083800 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -890,6 +890,11 @@ static void spapr_dt_rtas(SpaprMachineState *spapr, void
*fdt)
add_str(hypertas, "hcall-copy");
add_str(hypertas, "hcall-debug");
add_str(hypertas, "hcall-vphn");
+if (kvm_enabled() &&
+(spapr_get_cap(spapr, SPAPR_CAP_RPT_INVALIDATE) == SPAPR_CAP_ON)) {
+add_str(hypertas, "hcall-rpt-invalidate");
+}
+
add_str(qemu_hypertas, "hcall-memop1");
if (!kvm_enabled() || kvmppc_spapr_use_multitce()) {
@@ -2021,6 +2026,7 @@ static const VMStateDescription vmstate_spapr = {
_spapr_cap_ccf_assist,
_spapr_cap_fwnmi,
_spapr_fwnmi,
+_spapr_cap_rpt_invalidate,
NULL
}
};
@@ -4478,6 +4484,7 @@ static void spapr_machine_class_init(ObjectClass *oc,
void *data)
smc->default_caps.caps[SPAPR_CAP_LARGE_DECREMENTER] = SPAPR_CAP_ON;
smc->default_caps.caps[SPAPR_CAP_CCF_ASSIST] = SPAPR_CAP_ON;
smc->default_caps.caps[SPAPR_CAP_FWNMI] = SPAPR_CAP_ON;
+smc->default_caps.caps[SPAPR_CAP_RPT_INVALIDATE] = SPAPR_CAP_OFF;
spapr_caps_add_properties(smc);
smc->irq = _irq_dual;
smc->dr_phb_enabled = true;
diff --git a/hw/ppc/spapr_caps.c b/hw/ppc/spapr_caps.c
index 9341e9782a..ebf81e3b23 100644
--- a/hw/ppc/spapr_caps.c
+++ b/hw/ppc/spapr_caps.c
@@ -524,6 +524,45 @@ static void cap_fwnmi_apply(SpaprMachineState *spapr,
uint8_t val,
}
}
+static void cap_rpt_invalidate_apply(SpaprMachineState *spapr,
+ uint8_t val, Error **errp)
+{
+ERRP_GUARD();
+PowerPCCPU *cpu = POWERPC_CPU(first_cpu);
+
+if (!val) {
+/* capability disabled by default */
+return;
+}
+
+if (tcg_enabled()) {
+error_setg(errp, "No H_RPT_INVALIDATE support in TCG");
+error_append_hint(errp, "Try appending -machine
cap-rpt-invalidate=off\n");
+} else if (kvm_enabled()) {
+if (!ppc_check_compat(cpu, CPU_POWERPC_LOGICAL_3_00, 0,
+ spapr->max_compat_pvr)) {
+error_setg(errp, "H_RPT_INVALIDATE only supported on POWER9");
+error_append_hint(errp,
+ "Try appending -machine
max-cpu-compat=power9\n");
+return;
+}
+
+if (!kvmppc_has_cap_mmu_radix()) {
+error_setg(errp, "H_RPT_INVALIDATE only supported on Radix");
+return;
+}
+
+if (!kvmppc_has_cap_rpt_invalidate()) {
+error_setg(errp,
+ "KVM implementation does not support H_RPT_INVALIDATE");
+error_append_hint(errp,
+ "Try appending -machine
cap-rpt-invalidate=off\n");
+} else {
+kvmppc_enable_h_rpt_invalidate();
+}
+}
+}
+
SpaprCapabilityInfo capability_table[SPAPR_CAP_NUM] = {
[SPAPR_CAP_HTM] = {
.name = "htm",
@@ -632,6 +671,15 @@ SpaprCapabilityInfo capability_table[SPAPR_CAP_NUM] = {
.type = "bool",
.apply = cap_fwnmi_apply,
},
+[SPAPR_CAP_RPT_INVALIDATE] = {
+.name = "rpt-invalidate",
+.description = "Allow H_RPT_INVALIDATE",
+.index = SPAPR_CAP_RPT_INVALIDATE,
+.get = spapr_cap_get_bool,
+
[RFC PATCH v0 1/1] target/ppc: Support for H_RPT_INVALIDATE hcall
If KVM_CAP_RPT_INVALIDATE KVM capability is enabled, then
- indicate the availability of H_RPT_INVALIDATE hcall to the guest via
ibm,hypertas-functions property.
- Enable the hcall
Both the above are done only if the new sPAPR machine capability
cap-rpt-invalidate is set.
Note: The KVM implementation of the hcall has been posted for upstream
review here:
https://lore.kernel.org/linuxppc-dev/20210105090557.2150104-1-bhar...@linux.ibm.com/T/#t
Update to linux-headers/linux/kvm.h here is temporary, will be
done via header updates once the kernel change is accepted upstream.
Signed-off-by: Bharata B Rao
---
hw/ppc/spapr.c| 7 ++
hw/ppc/spapr_caps.c | 49 +++
include/hw/ppc/spapr.h| 8 +--
linux-headers/linux/kvm.h | 1 +
target/ppc/kvm.c | 12 ++
target/ppc/kvm_ppc.h | 11 +
6 files changed, 86 insertions(+), 2 deletions(-)
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index 489cefcb81..0228083800 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -890,6 +890,11 @@ static void spapr_dt_rtas(SpaprMachineState *spapr, void
*fdt)
add_str(hypertas, "hcall-copy");
add_str(hypertas, "hcall-debug");
add_str(hypertas, "hcall-vphn");
+if (kvm_enabled() &&
+(spapr_get_cap(spapr, SPAPR_CAP_RPT_INVALIDATE) == SPAPR_CAP_ON)) {
+add_str(hypertas, "hcall-rpt-invalidate");
+}
+
add_str(qemu_hypertas, "hcall-memop1");
if (!kvm_enabled() || kvmppc_spapr_use_multitce()) {
@@ -2021,6 +2026,7 @@ static const VMStateDescription vmstate_spapr = {
_spapr_cap_ccf_assist,
_spapr_cap_fwnmi,
_spapr_fwnmi,
+_spapr_cap_rpt_invalidate,
NULL
}
};
@@ -4478,6 +4484,7 @@ static void spapr_machine_class_init(ObjectClass *oc,
void *data)
smc->default_caps.caps[SPAPR_CAP_LARGE_DECREMENTER] = SPAPR_CAP_ON;
smc->default_caps.caps[SPAPR_CAP_CCF_ASSIST] = SPAPR_CAP_ON;
smc->default_caps.caps[SPAPR_CAP_FWNMI] = SPAPR_CAP_ON;
+smc->default_caps.caps[SPAPR_CAP_RPT_INVALIDATE] = SPAPR_CAP_OFF;
spapr_caps_add_properties(smc);
smc->irq = _irq_dual;
smc->dr_phb_enabled = true;
diff --git a/hw/ppc/spapr_caps.c b/hw/ppc/spapr_caps.c
index 9341e9782a..ebf81e3b23 100644
--- a/hw/ppc/spapr_caps.c
+++ b/hw/ppc/spapr_caps.c
@@ -524,6 +524,45 @@ static void cap_fwnmi_apply(SpaprMachineState *spapr,
uint8_t val,
}
}
+static void cap_rpt_invalidate_apply(SpaprMachineState *spapr,
+ uint8_t val, Error **errp)
+{
+ERRP_GUARD();
+PowerPCCPU *cpu = POWERPC_CPU(first_cpu);
+
+if (!val) {
+/* capability disabled by default */
+return;
+}
+
+if (tcg_enabled()) {
+error_setg(errp, "No H_RPT_INVALIDATE support in TCG");
+error_append_hint(errp, "Try appending -machine
cap-rpt-invalidate=off\n");
+} else if (kvm_enabled()) {
+if (!ppc_check_compat(cpu, CPU_POWERPC_LOGICAL_3_00, 0,
+ spapr->max_compat_pvr)) {
+error_setg(errp, "H_RPT_INVALIDATE only supported on POWER9");
+error_append_hint(errp,
+ "Try appending -machine
max-cpu-compat=power9\n");
+return;
+}
+
+if (!kvmppc_has_cap_mmu_radix()) {
+error_setg(errp, "H_RPT_INVALIDATE only supported on Radix");
+return;
+}
+
+if (!kvmppc_has_cap_rpt_invalidate()) {
+error_setg(errp,
+ "KVM implementation does not support H_RPT_INVALIDATE");
+error_append_hint(errp,
+ "Try appending -machine
cap-rpt-invalidate=off\n");
+} else {
+kvmppc_enable_h_rpt_invalidate();
+}
+}
+}
+
SpaprCapabilityInfo capability_table[SPAPR_CAP_NUM] = {
[SPAPR_CAP_HTM] = {
.name = "htm",
@@ -632,6 +671,15 @@ SpaprCapabilityInfo capability_table[SPAPR_CAP_NUM] = {
.type = "bool",
.apply = cap_fwnmi_apply,
},
+[SPAPR_CAP_RPT_INVALIDATE] = {
+.name = "rpt-invalidate",
+.description = "Allow H_RPT_INVALIDATE",
+.index = SPAPR_CAP_RPT_INVALIDATE,
+.get = spapr_cap_get_bool,
+.set = spapr_cap_set_bool,
+.type = "bool",
+.apply = cap_rpt_invalidate_apply,
+},
};
static SpaprCapabilities default_caps_with_cpu(SpaprMachineState *spapr,
@@ -772,6 +820,7 @@ SPAPR_CAP_MIG_STATE(nested_kvm_hv, SPAPR_CAP_NESTED_KVM_HV);
SPAPR_CAP_MIG_STATE(large_decr, SPAPR_CAP_LARGE_DECREMENTER);
SPAPR_CAP_MIG_STATE(ccf_assist, SPAPR_CAP_CCF_ASSIST);
SPAPR_CAP_MIG_STATE(fwnmi, SPAPR_CAP_FWNMI);
+SPAPR_CAP_MIG_STATE(rpt_invalidate, SPAPR_CAP_RPT_INVALIDATE);
void spapr_caps_init(SpaprMachineState *spapr)
{
diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h
index e0f10f252c..0931b5b8e8 100644
--- a/include/hw/ppc/spapr.h
+++ b/include/hw/ppc/spapr.h
@@ -74,8
