Re: [PATCH 9/9] spapr: implement nested-hv capability for the virtual hypervisor
On 2/16/22 02:16, Nicholas Piggin wrote:
Excerpts from Cédric Le Goater's message of February 16, 2022 4:21 am:
On 2/15/22 04:16, Nicholas Piggin wrote:
This implements the Nested KVM HV hcall API for spapr under TCG.
The L2 is switched in when the H_ENTER_NESTED hcall is made, and the
L1 is switched back in returned from the hcall when a HV exception
is sent to the vhyp. Register state is copied in and out according to
the nested KVM HV hcall API specification.
The hdecr timer is started when the L2 is switched in, and it provides
the HDEC / 0x980 return to L1.
The MMU re-uses the bare metal radix 2-level page table walker by
using the get_pate method to point the MMU to the nested partition
table entry. MMU faults due to partition scope errors raise HV
exceptions and accordingly are routed back to the L1.
The MMU does not tag translations for the L1 (direct) vs L2 (nested)
guests, so the TLB is flushed on any L1<->L2 transition (hcall entry
and exit).
Signed-off-by: Nicholas Piggin
---
hw/ppc/spapr.c | 32 +++-
hw/ppc/spapr_caps.c| 11 +-
hw/ppc/spapr_hcall.c | 321 +
include/hw/ppc/spapr.h | 74 +-
target/ppc/cpu.h | 3 +
5 files changed, 431 insertions(+), 10 deletions(-)
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index 3a5cf92c94..6988e3ec76 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -1314,11 +1314,32 @@ static bool spapr_get_pate(PPCVirtualHypervisor *vhyp,
PowerPCCPU *cpu,
{
SpaprMachineState *spapr = SPAPR_MACHINE(vhyp);
-assert(lpid == 0);
+if (!cpu->in_spapr_nested) {
Since 'in_spapr_nested' is a spapr CPU characteristic, I don't think
it belongs to PowerPCCPU. See the end of the patch, for a proposal.
SpaprCpuState. Certainly that's a better place, I must have missed it.
btw, this helps the ordering of files :
[diff]
orderFile = /path/to/qemu/scripts/git.orderfile
+assert(lpid == 0);
-/* Copy PATE1:GR into PATE0:HR */
-entry->dw0 = spapr->patb_entry & PATE0_HR;
-entry->dw1 = spapr->patb_entry;
+/* Copy PATE1:GR into PATE0:HR */
+entry->dw0 = spapr->patb_entry & PATE0_HR;
+entry->dw1 = spapr->patb_entry;
+
+} else {
+uint64_t patb, pats;
+
+assert(lpid != 0);
+
+patb = spapr->nested_ptcr & PTCR_PATB;
+pats = spapr->nested_ptcr & PTCR_PATS;
+
+/* Calculate number of entries */
+pats = 1ull << (pats + 12 - 4);
+if (pats <= lpid) {
+return false;
+}
+
+/* Grab entry */
+patb += 16 * lpid;
+entry->dw0 = ldq_phys(CPU(cpu)->as, patb);
+entry->dw1 = ldq_phys(CPU(cpu)->as, patb + 8);
+}
return true;
}
@@ -4472,7 +4493,7 @@ PowerPCCPU *spapr_find_cpu(int vcpu_id)
static bool spapr_cpu_in_nested(PowerPCCPU *cpu)
{
-return false;
+return cpu->in_spapr_nested;
}
static void spapr_cpu_exec_enter(PPCVirtualHypervisor *vhyp, PowerPCCPU *cpu)
@@ -4584,6 +4605,7 @@ static void spapr_machine_class_init(ObjectClass *oc,
void *data)
nc->nmi_monitor_handler = spapr_nmi;
smc->phb_placement = spapr_phb_placement;
vhc->cpu_in_nested = spapr_cpu_in_nested;
+vhc->deliver_hv_excp = spapr_exit_nested;
vhc->hypercall = emulate_spapr_hypercall;
vhc->hpt_mask = spapr_hpt_mask;
vhc->map_hptes = spapr_map_hptes;
diff --git a/hw/ppc/spapr_caps.c b/hw/ppc/spapr_caps.c
index 5cc80776d0..4d8bb2ad2c 100644
--- a/hw/ppc/spapr_caps.c
+++ b/hw/ppc/spapr_caps.c
@@ -444,19 +444,22 @@ static void cap_nested_kvm_hv_apply(SpaprMachineState
*spapr,
{
ERRP_GUARD();
PowerPCCPU *cpu = POWERPC_CPU(first_cpu);
+CPUPPCState *env = >env;
if (!val) {
/* capability disabled by default */
return;
}
-if (tcg_enabled()) {
-error_setg(errp, "No Nested KVM-HV support in TCG");
I don't like using KVM-HV (which is KVM-over-PowerNV) when talking about
KVM-over-pseries. I think the platform name is important. Anyhow, this is
a more global discussion but we should talk about it someday because these
HV mode are becoming confusing ! We have PR also :)
The cap is nested-hv and QEMU describes it nested KVM HV. Are we stuck
with that? That could make a name change even more confusing.
It's really a new backend for the KVM HV front end. Like how POWER8 /
POWER9 bare metal backends are completely different now.
But I guess that does not help the end user to understand. On the other
hand, the user might not think "HV" is the HV mode of the CPU and just
thinks of it as "hypervisor".
I like paravirt-hv but nested-hv is not too bad. Anyway I'm happy to
change it.
+if (!(env->insns_flags2 & PPC2_ISA300)) {
+error_setg(errp, "Nested KVM-HV only supported on POWER9 and later");
error_append_hint(errp, "Try appending -machine
cap-nested-hv=off\n");
return ?
Re: [PATCH 9/9] spapr: implement nested-hv capability for the virtual hypervisor
Excerpts from Cédric Le Goater's message of February 16, 2022 4:21 am:
> On 2/15/22 04:16, Nicholas Piggin wrote:
>> This implements the Nested KVM HV hcall API for spapr under TCG.
>>
>> The L2 is switched in when the H_ENTER_NESTED hcall is made, and the
>> L1 is switched back in returned from the hcall when a HV exception
>> is sent to the vhyp. Register state is copied in and out according to
>> the nested KVM HV hcall API specification.
>>
>> The hdecr timer is started when the L2 is switched in, and it provides
>> the HDEC / 0x980 return to L1.
>>
>> The MMU re-uses the bare metal radix 2-level page table walker by
>> using the get_pate method to point the MMU to the nested partition
>> table entry. MMU faults due to partition scope errors raise HV
>> exceptions and accordingly are routed back to the L1.
>>
>> The MMU does not tag translations for the L1 (direct) vs L2 (nested)
>> guests, so the TLB is flushed on any L1<->L2 transition (hcall entry
>> and exit).
>>
>> Signed-off-by: Nicholas Piggin
>> ---
>> hw/ppc/spapr.c | 32 +++-
>> hw/ppc/spapr_caps.c| 11 +-
>> hw/ppc/spapr_hcall.c | 321 +
>> include/hw/ppc/spapr.h | 74 +-
>> target/ppc/cpu.h | 3 +
>> 5 files changed, 431 insertions(+), 10 deletions(-)
>>
>> diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
>> index 3a5cf92c94..6988e3ec76 100644
>> --- a/hw/ppc/spapr.c
>> +++ b/hw/ppc/spapr.c
>> @@ -1314,11 +1314,32 @@ static bool spapr_get_pate(PPCVirtualHypervisor
>> *vhyp, PowerPCCPU *cpu,
>> {
>> SpaprMachineState *spapr = SPAPR_MACHINE(vhyp);
>>
>> -assert(lpid == 0);
>> +if (!cpu->in_spapr_nested) {
>
> Since 'in_spapr_nested' is a spapr CPU characteristic, I don't think
> it belongs to PowerPCCPU. See the end of the patch, for a proposal.
SpaprCpuState. Certainly that's a better place, I must have missed it.
>
> btw, this helps the ordering of files :
>
> [diff]
> orderFile = /path/to/qemu/scripts/git.orderfile
>
>> +assert(lpid == 0);
>>
>> -/* Copy PATE1:GR into PATE0:HR */
>> -entry->dw0 = spapr->patb_entry & PATE0_HR;
>> -entry->dw1 = spapr->patb_entry;
>> +/* Copy PATE1:GR into PATE0:HR */
>> +entry->dw0 = spapr->patb_entry & PATE0_HR;
>> +entry->dw1 = spapr->patb_entry;
>> +
>> +} else {
>> +uint64_t patb, pats;
>> +
>> +assert(lpid != 0);
>> +
>> +patb = spapr->nested_ptcr & PTCR_PATB;
>> +pats = spapr->nested_ptcr & PTCR_PATS;
>> +
>> +/* Calculate number of entries */
>> +pats = 1ull << (pats + 12 - 4);
>> +if (pats <= lpid) {
>> +return false;
>> +}
>> +
>> +/* Grab entry */
>> +patb += 16 * lpid;
>> +entry->dw0 = ldq_phys(CPU(cpu)->as, patb);
>> +entry->dw1 = ldq_phys(CPU(cpu)->as, patb + 8);
>> +}
>>
>> return true;
>> }
>> @@ -4472,7 +4493,7 @@ PowerPCCPU *spapr_find_cpu(int vcpu_id)
>>
>> static bool spapr_cpu_in_nested(PowerPCCPU *cpu)
>> {
>> -return false;
>> +return cpu->in_spapr_nested;
>> }
>>
>> static void spapr_cpu_exec_enter(PPCVirtualHypervisor *vhyp, PowerPCCPU
>> *cpu)
>> @@ -4584,6 +4605,7 @@ static void spapr_machine_class_init(ObjectClass *oc,
>> void *data)
>> nc->nmi_monitor_handler = spapr_nmi;
>> smc->phb_placement = spapr_phb_placement;
>> vhc->cpu_in_nested = spapr_cpu_in_nested;
>> +vhc->deliver_hv_excp = spapr_exit_nested;
>> vhc->hypercall = emulate_spapr_hypercall;
>> vhc->hpt_mask = spapr_hpt_mask;
>> vhc->map_hptes = spapr_map_hptes;
>> diff --git a/hw/ppc/spapr_caps.c b/hw/ppc/spapr_caps.c
>> index 5cc80776d0..4d8bb2ad2c 100644
>> --- a/hw/ppc/spapr_caps.c
>> +++ b/hw/ppc/spapr_caps.c
>> @@ -444,19 +444,22 @@ static void cap_nested_kvm_hv_apply(SpaprMachineState
>> *spapr,
>> {
>> ERRP_GUARD();
>> PowerPCCPU *cpu = POWERPC_CPU(first_cpu);
>> +CPUPPCState *env = >env;
>>
>> if (!val) {
>> /* capability disabled by default */
>> return;
>> }
>>
>> -if (tcg_enabled()) {
>> -error_setg(errp, "No Nested KVM-HV support in TCG");
>
> I don't like using KVM-HV (which is KVM-over-PowerNV) when talking about
> KVM-over-pseries. I think the platform name is important. Anyhow, this is
> a more global discussion but we should talk about it someday because these
> HV mode are becoming confusing ! We have PR also :)
The cap is nested-hv and QEMU describes it nested KVM HV. Are we stuck
with that? That could make a name change even more confusing.
It's really a new backend for the KVM HV front end. Like how POWER8 /
POWER9 bare metal backends are completely different now.
But I guess that does not help the end user to understand. On the other
hand, the user might not think "HV" is the HV mode of the CPU and just
thinks of it as "hypervisor".
I like paravirt-hv but nested-hv is not too bad.
Re: [PATCH 9/9] spapr: implement nested-hv capability for the virtual hypervisor
On 2/15/22 04:16, Nicholas Piggin wrote:
This implements the Nested KVM HV hcall API for spapr under TCG.
The L2 is switched in when the H_ENTER_NESTED hcall is made, and the
L1 is switched back in returned from the hcall when a HV exception
is sent to the vhyp. Register state is copied in and out according to
the nested KVM HV hcall API specification.
The hdecr timer is started when the L2 is switched in, and it provides
the HDEC / 0x980 return to L1.
The MMU re-uses the bare metal radix 2-level page table walker by
using the get_pate method to point the MMU to the nested partition
table entry. MMU faults due to partition scope errors raise HV
exceptions and accordingly are routed back to the L1.
The MMU does not tag translations for the L1 (direct) vs L2 (nested)
guests, so the TLB is flushed on any L1<->L2 transition (hcall entry
and exit).
Signed-off-by: Nicholas Piggin
---
hw/ppc/spapr.c | 32 +++-
hw/ppc/spapr_caps.c| 11 +-
hw/ppc/spapr_hcall.c | 321 +
include/hw/ppc/spapr.h | 74 +-
target/ppc/cpu.h | 3 +
5 files changed, 431 insertions(+), 10 deletions(-)
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index 3a5cf92c94..6988e3ec76 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -1314,11 +1314,32 @@ static bool spapr_get_pate(PPCVirtualHypervisor *vhyp,
PowerPCCPU *cpu,
{
SpaprMachineState *spapr = SPAPR_MACHINE(vhyp);
-assert(lpid == 0);
+if (!cpu->in_spapr_nested) {
Since 'in_spapr_nested' is a spapr CPU characteristic, I don't think
it belongs to PowerPCCPU. See the end of the patch, for a proposal.
btw, this helps the ordering of files :
[diff]
orderFile = /path/to/qemu/scripts/git.orderfile
+assert(lpid == 0);
-/* Copy PATE1:GR into PATE0:HR */
-entry->dw0 = spapr->patb_entry & PATE0_HR;
-entry->dw1 = spapr->patb_entry;
+/* Copy PATE1:GR into PATE0:HR */
+entry->dw0 = spapr->patb_entry & PATE0_HR;
+entry->dw1 = spapr->patb_entry;
+
+} else {
+uint64_t patb, pats;
+
+assert(lpid != 0);
+
+patb = spapr->nested_ptcr & PTCR_PATB;
+pats = spapr->nested_ptcr & PTCR_PATS;
+
+/* Calculate number of entries */
+pats = 1ull << (pats + 12 - 4);
+if (pats <= lpid) {
+return false;
+}
+
+/* Grab entry */
+patb += 16 * lpid;
+entry->dw0 = ldq_phys(CPU(cpu)->as, patb);
+entry->dw1 = ldq_phys(CPU(cpu)->as, patb + 8);
+}
return true;
}
@@ -4472,7 +4493,7 @@ PowerPCCPU *spapr_find_cpu(int vcpu_id)
static bool spapr_cpu_in_nested(PowerPCCPU *cpu)
{
-return false;
+return cpu->in_spapr_nested;
}
static void spapr_cpu_exec_enter(PPCVirtualHypervisor *vhyp, PowerPCCPU *cpu)
@@ -4584,6 +4605,7 @@ static void spapr_machine_class_init(ObjectClass *oc,
void *data)
nc->nmi_monitor_handler = spapr_nmi;
smc->phb_placement = spapr_phb_placement;
vhc->cpu_in_nested = spapr_cpu_in_nested;
+vhc->deliver_hv_excp = spapr_exit_nested;
vhc->hypercall = emulate_spapr_hypercall;
vhc->hpt_mask = spapr_hpt_mask;
vhc->map_hptes = spapr_map_hptes;
diff --git a/hw/ppc/spapr_caps.c b/hw/ppc/spapr_caps.c
index 5cc80776d0..4d8bb2ad2c 100644
--- a/hw/ppc/spapr_caps.c
+++ b/hw/ppc/spapr_caps.c
@@ -444,19 +444,22 @@ static void cap_nested_kvm_hv_apply(SpaprMachineState
*spapr,
{
ERRP_GUARD();
PowerPCCPU *cpu = POWERPC_CPU(first_cpu);
+CPUPPCState *env = >env;
if (!val) {
/* capability disabled by default */
return;
}
-if (tcg_enabled()) {
-error_setg(errp, "No Nested KVM-HV support in TCG");
I don't like using KVM-HV (which is KVM-over-PowerNV) when talking about
KVM-over-pseries. I think the platform name is important. Anyhow, this is
a more global discussion but we should talk about it someday because these
HV mode are becoming confusing ! We have PR also :)
+if (!(env->insns_flags2 & PPC2_ISA300)) {
+error_setg(errp, "Nested KVM-HV only supported on POWER9 and later");
error_append_hint(errp, "Try appending -machine cap-nested-hv=off\n");
return ?
-} else if (kvm_enabled()) {
+}
+
+if (kvm_enabled()) {
if (!ppc_check_compat(cpu, CPU_POWERPC_LOGICAL_3_00, 0,
spapr->max_compat_pvr)) {
-error_setg(errp, "Nested KVM-HV only supported on POWER9");
+error_setg(errp, "Nested KVM-HV only supported on POWER9 and
later");
error_append_hint(errp,
"Try appending -machine
max-cpu-compat=power9\n");
return;
diff --git a/hw/ppc/spapr_hcall.c b/hw/ppc/spapr_hcall.c
index 5dec056796..3129fae90d 100644
--- a/hw/ppc/spapr_hcall.c
+++ b/hw/ppc/spapr_hcall.c
@@ -9,6 +9,7 @@
#include "qemu/error-report.h"
#include "exec/exec-all.h"
#include "helper_regs.h"
Re: [PATCH 9/9] spapr: implement nested-hv capability for the virtual hypervisor
Nicholas Piggin writes: > This implements the Nested KVM HV hcall API for spapr under TCG. > > The L2 is switched in when the H_ENTER_NESTED hcall is made, and the > L1 is switched back in returned from the hcall when a HV exception > is sent to the vhyp. Register state is copied in and out according to > the nested KVM HV hcall API specification. > > The hdecr timer is started when the L2 is switched in, and it provides > the HDEC / 0x980 return to L1. > > The MMU re-uses the bare metal radix 2-level page table walker by > using the get_pate method to point the MMU to the nested partition > table entry. MMU faults due to partition scope errors raise HV > exceptions and accordingly are routed back to the L1. > > The MMU does not tag translations for the L1 (direct) vs L2 (nested) > guests, so the TLB is flushed on any L1<->L2 transition (hcall entry > and exit). > > Signed-off-by: Nicholas Piggin Reviewed-by: Fabiano Rosas
[PATCH 9/9] spapr: implement nested-hv capability for the virtual hypervisor
This implements the Nested KVM HV hcall API for spapr under TCG.
The L2 is switched in when the H_ENTER_NESTED hcall is made, and the
L1 is switched back in returned from the hcall when a HV exception
is sent to the vhyp. Register state is copied in and out according to
the nested KVM HV hcall API specification.
The hdecr timer is started when the L2 is switched in, and it provides
the HDEC / 0x980 return to L1.
The MMU re-uses the bare metal radix 2-level page table walker by
using the get_pate method to point the MMU to the nested partition
table entry. MMU faults due to partition scope errors raise HV
exceptions and accordingly are routed back to the L1.
The MMU does not tag translations for the L1 (direct) vs L2 (nested)
guests, so the TLB is flushed on any L1<->L2 transition (hcall entry
and exit).
Signed-off-by: Nicholas Piggin
---
hw/ppc/spapr.c | 32 +++-
hw/ppc/spapr_caps.c| 11 +-
hw/ppc/spapr_hcall.c | 321 +
include/hw/ppc/spapr.h | 74 +-
target/ppc/cpu.h | 3 +
5 files changed, 431 insertions(+), 10 deletions(-)
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index 3a5cf92c94..6988e3ec76 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -1314,11 +1314,32 @@ static bool spapr_get_pate(PPCVirtualHypervisor *vhyp,
PowerPCCPU *cpu,
{
SpaprMachineState *spapr = SPAPR_MACHINE(vhyp);
-assert(lpid == 0);
+if (!cpu->in_spapr_nested) {
+assert(lpid == 0);
-/* Copy PATE1:GR into PATE0:HR */
-entry->dw0 = spapr->patb_entry & PATE0_HR;
-entry->dw1 = spapr->patb_entry;
+/* Copy PATE1:GR into PATE0:HR */
+entry->dw0 = spapr->patb_entry & PATE0_HR;
+entry->dw1 = spapr->patb_entry;
+
+} else {
+uint64_t patb, pats;
+
+assert(lpid != 0);
+
+patb = spapr->nested_ptcr & PTCR_PATB;
+pats = spapr->nested_ptcr & PTCR_PATS;
+
+/* Calculate number of entries */
+pats = 1ull << (pats + 12 - 4);
+if (pats <= lpid) {
+return false;
+}
+
+/* Grab entry */
+patb += 16 * lpid;
+entry->dw0 = ldq_phys(CPU(cpu)->as, patb);
+entry->dw1 = ldq_phys(CPU(cpu)->as, patb + 8);
+}
return true;
}
@@ -4472,7 +4493,7 @@ PowerPCCPU *spapr_find_cpu(int vcpu_id)
static bool spapr_cpu_in_nested(PowerPCCPU *cpu)
{
-return false;
+return cpu->in_spapr_nested;
}
static void spapr_cpu_exec_enter(PPCVirtualHypervisor *vhyp, PowerPCCPU *cpu)
@@ -4584,6 +4605,7 @@ static void spapr_machine_class_init(ObjectClass *oc,
void *data)
nc->nmi_monitor_handler = spapr_nmi;
smc->phb_placement = spapr_phb_placement;
vhc->cpu_in_nested = spapr_cpu_in_nested;
+vhc->deliver_hv_excp = spapr_exit_nested;
vhc->hypercall = emulate_spapr_hypercall;
vhc->hpt_mask = spapr_hpt_mask;
vhc->map_hptes = spapr_map_hptes;
diff --git a/hw/ppc/spapr_caps.c b/hw/ppc/spapr_caps.c
index 5cc80776d0..4d8bb2ad2c 100644
--- a/hw/ppc/spapr_caps.c
+++ b/hw/ppc/spapr_caps.c
@@ -444,19 +444,22 @@ static void cap_nested_kvm_hv_apply(SpaprMachineState
*spapr,
{
ERRP_GUARD();
PowerPCCPU *cpu = POWERPC_CPU(first_cpu);
+CPUPPCState *env = >env;
if (!val) {
/* capability disabled by default */
return;
}
-if (tcg_enabled()) {
-error_setg(errp, "No Nested KVM-HV support in TCG");
+if (!(env->insns_flags2 & PPC2_ISA300)) {
+error_setg(errp, "Nested KVM-HV only supported on POWER9 and later");
error_append_hint(errp, "Try appending -machine cap-nested-hv=off\n");
-} else if (kvm_enabled()) {
+}
+
+if (kvm_enabled()) {
if (!ppc_check_compat(cpu, CPU_POWERPC_LOGICAL_3_00, 0,
spapr->max_compat_pvr)) {
-error_setg(errp, "Nested KVM-HV only supported on POWER9");
+error_setg(errp, "Nested KVM-HV only supported on POWER9 and
later");
error_append_hint(errp,
"Try appending -machine
max-cpu-compat=power9\n");
return;
diff --git a/hw/ppc/spapr_hcall.c b/hw/ppc/spapr_hcall.c
index 5dec056796..3129fae90d 100644
--- a/hw/ppc/spapr_hcall.c
+++ b/hw/ppc/spapr_hcall.c
@@ -9,6 +9,7 @@
#include "qemu/error-report.h"
#include "exec/exec-all.h"
#include "helper_regs.h"
+#include "hw/ppc/ppc.h"
#include "hw/ppc/spapr.h"
#include "hw/ppc/spapr_cpu_core.h"
#include "mmu-hash64.h"
@@ -1500,6 +1501,321 @@ static void hypercall_register_softmmu(void)
}
#endif
+/* TCG only */
+#define PRTS_MASK 0x1f
+
+static target_ulong h_set_ptbl(PowerPCCPU *cpu,
+ SpaprMachineState *spapr,
+ target_ulong opcode,
+ target_ulong *args)
+{
+target_ulong ptcr = args[0];
+
+if (!spapr_get_cap(spapr, SPAPR_CAP_NESTED_KVM_HV)) {
+return H_FUNCTION;
+}
+
+if ((ptcr & PRTS_MASK) + 12 - 4 >
