[PATCH v3 2/4] s390x: Add missing vcpu reset functions
Up to now we only had an ioctl to reset vcpu data QEMU couldn't reach for the initial reset, and that was also called for the clear reset. To be architecture compliant, we also need to clear local interrupts on a normal reset. Because of this and the upcoming protvirt support we need to add ioctls for the missing clear and normal resets. Signed-off-by: Janosch Frank Reviewed-by: Thomas Huth Acked-by: David Hildenbrand --- target/s390x/cpu.c | 16 +-- target/s390x/kvm-stub.c | 10 +- target/s390x/kvm.c | 42 target/s390x/kvm_s390x.h | 4 +++- 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c index 829ce6ad54..4973365d6c 100644 --- a/target/s390x/cpu.c +++ b/target/s390x/cpu.c @@ -139,8 +139,20 @@ static void s390_cpu_reset(CPUState *s, cpu_reset_type type) } /* Reset state inside the kernel that we cannot access yet from QEMU. */ -if (kvm_enabled() && type != S390_CPU_RESET_NORMAL) { -kvm_s390_reset_vcpu(cpu); +if (kvm_enabled()) { +switch (type) { +case S390_CPU_RESET_CLEAR: +kvm_s390_reset_vcpu_clear(cpu); +break; +case S390_CPU_RESET_INITIAL: +kvm_s390_reset_vcpu_initial(cpu); +break; +case S390_CPU_RESET_NORMAL: +kvm_s390_reset_vcpu_normal(cpu); +break; +default: +g_assert_not_reached(); +} } } diff --git a/target/s390x/kvm-stub.c b/target/s390x/kvm-stub.c index 5152e2bdf1..c4cd497f85 100644 --- a/target/s390x/kvm-stub.c +++ b/target/s390x/kvm-stub.c @@ -83,7 +83,15 @@ void kvm_s390_cmma_reset(void) { } -void kvm_s390_reset_vcpu(S390CPU *cpu) +void kvm_s390_reset_vcpu_initial(S390CPU *cpu) +{ +} + +void kvm_s390_reset_vcpu_clear(S390CPU *cpu) +{ +} + +void kvm_s390_reset_vcpu_normal(S390CPU *cpu) { } diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c index ad6e38c876..f633472980 100644 --- a/target/s390x/kvm.c +++ b/target/s390x/kvm.c @@ -151,6 +151,7 @@ static int cap_s390_irq; static int cap_ri; static int cap_gs; static int cap_hpage_1m; +static int cap_vcpu_resets; static int active_cmma; @@ -342,6 +343,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s) cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF); cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP); cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ); +cap_vcpu_resets = kvm_check_extension(s, KVM_CAP_S390_VCPU_RESETS); if (!kvm_check_extension(s, KVM_CAP_S390_GMAP) || !kvm_check_extension(s, KVM_CAP_S390_COW)) { @@ -403,17 +405,41 @@ int kvm_arch_destroy_vcpu(CPUState *cs) return 0; } -void kvm_s390_reset_vcpu(S390CPU *cpu) +static void kvm_s390_reset_vcpu(S390CPU *cpu, unsigned long type) { CPUState *cs = CPU(cpu); -/* The initial reset call is needed here to reset in-kernel - * vcpu data that we can't access directly from QEMU - * (i.e. with older kernels which don't support sync_regs/ONE_REG). - * Before this ioctl cpu_synchronize_state() is called in common kvm - * code (kvm-all) */ -if (kvm_vcpu_ioctl(cs, KVM_S390_INITIAL_RESET, NULL)) { -error_report("Initial CPU reset failed on CPU %i", cs->cpu_index); +/* + * The reset call is needed here to reset in-kernel vcpu data that + * we can't access directly from QEMU (i.e. with older kernels + * which don't support sync_regs/ONE_REG). Before this ioctl + * cpu_synchronize_state() is called in common kvm code + * (kvm-all). + */ +if (kvm_vcpu_ioctl(cs, type)) { +error_report("CPU reset failed on CPU %i type %lx", + cs->cpu_index, type); +} +} + +void kvm_s390_reset_vcpu_initial(S390CPU *cpu) +{ +kvm_s390_reset_vcpu(cpu, KVM_S390_INITIAL_RESET); +} + +void kvm_s390_reset_vcpu_clear(S390CPU *cpu) +{ +if (cap_vcpu_resets) { +kvm_s390_reset_vcpu(cpu, KVM_S390_CLEAR_RESET); +} else { +kvm_s390_reset_vcpu(cpu, KVM_S390_INITIAL_RESET); +} +} + +void kvm_s390_reset_vcpu_normal(S390CPU *cpu) +{ +if (cap_vcpu_resets) { +kvm_s390_reset_vcpu(cpu, KVM_S390_NORMAL_RESET); } } diff --git a/target/s390x/kvm_s390x.h b/target/s390x/kvm_s390x.h index caf985955b..0b21789796 100644 --- a/target/s390x/kvm_s390x.h +++ b/target/s390x/kvm_s390x.h @@ -34,7 +34,9 @@ int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch, int vq, bool assign); int kvm_s390_cmma_active(void); void kvm_s390_cmma_reset(void); -void kvm_s390_reset_vcpu(S390CPU *cpu); +void kvm_s390_reset_vcpu_clear(S390CPU *cpu); +void kvm_s390_reset_vcpu_normal(S390CPU *cpu); +void kvm_s390_reset_vcpu_initial(S390CPU *cpu); int kvm_s390_set_mem_limit(uint64_t new_limit, uint64_t *hw_limit); void kvm_s390_set_max_pagesize(uint64_t pagesize, Error **errp); void
Re: [PATCH v3 2/4] s390x: Add missing vcpu reset functions
On Tue, 3 Dec 2019 08:28:11 -0500 Janosch Frank wrote: > Up to now we only had an ioctl to reset vcpu data QEMU couldn't reach > for the initial reset, and that was also called for the clear > reset. To be architecture compliant, we also need to clear local > interrupts on a normal reset. > > Because of this and the upcoming protvirt support we need to add > ioctls for the missing clear and normal resets. > > Signed-off-by: Janosch Frank > Reviewed-by: Thomas Huth > Acked-by: David Hildenbrand > --- > target/s390x/cpu.c | 16 +-- > target/s390x/kvm-stub.c | 10 +- > target/s390x/kvm.c | 42 > target/s390x/kvm_s390x.h | 4 +++- > 4 files changed, 60 insertions(+), 12 deletions(-) > > diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c > index 829ce6ad54..4973365d6c 100644 > --- a/target/s390x/cpu.c > +++ b/target/s390x/cpu.c > @@ -139,8 +139,20 @@ static void s390_cpu_reset(CPUState *s, cpu_reset_type > type) > } > > /* Reset state inside the kernel that we cannot access yet from QEMU. */ For the last iteration, I asked about the 'yet' here... > -if (kvm_enabled() && type != S390_CPU_RESET_NORMAL) { > -kvm_s390_reset_vcpu(cpu); > +if (kvm_enabled()) { > +switch (type) { > +case S390_CPU_RESET_CLEAR: > +kvm_s390_reset_vcpu_clear(cpu); > +break; > +case S390_CPU_RESET_INITIAL: > +kvm_s390_reset_vcpu_initial(cpu); > +break; > +case S390_CPU_RESET_NORMAL: > +kvm_s390_reset_vcpu_normal(cpu); > +break; > +default: > +g_assert_not_reached(); > +} > } > } > (...) > @@ -403,17 +405,41 @@ int kvm_arch_destroy_vcpu(CPUState *cs) > return 0; > } > > -void kvm_s390_reset_vcpu(S390CPU *cpu) > +static void kvm_s390_reset_vcpu(S390CPU *cpu, unsigned long type) > { > CPUState *cs = CPU(cpu); > > -/* The initial reset call is needed here to reset in-kernel > - * vcpu data that we can't access directly from QEMU > - * (i.e. with older kernels which don't support sync_regs/ONE_REG). > - * Before this ioctl cpu_synchronize_state() is called in common kvm > - * code (kvm-all) */ > -if (kvm_vcpu_ioctl(cs, KVM_S390_INITIAL_RESET, NULL)) { > -error_report("Initial CPU reset failed on CPU %i", cs->cpu_index); > +/* > + * The reset call is needed here to reset in-kernel vcpu data that > + * we can't access directly from QEMU (i.e. with older kernels > + * which don't support sync_regs/ONE_REG). Before this ioctl ...and this reference to 'older kernels' here. Are the comments still correct/relevant? > + * cpu_synchronize_state() is called in common kvm code > + * (kvm-all). > + */ > +if (kvm_vcpu_ioctl(cs, type)) { > +error_report("CPU reset failed on CPU %i type %lx", > + cs->cpu_index, type); > +} > +}
Re: [PATCH v3 2/4] s390x: Add missing vcpu reset functions
On 12/3/19 6:44 PM, Cornelia Huck wrote: > On Tue, 3 Dec 2019 08:28:11 -0500 > Janosch Frank wrote: > >> Up to now we only had an ioctl to reset vcpu data QEMU couldn't reach >> for the initial reset, and that was also called for the clear >> reset. To be architecture compliant, we also need to clear local >> interrupts on a normal reset. >> >> Because of this and the upcoming protvirt support we need to add >> ioctls for the missing clear and normal resets. >> >> Signed-off-by: Janosch Frank >> Reviewed-by: Thomas Huth >> Acked-by: David Hildenbrand >> --- >> target/s390x/cpu.c | 16 +-- >> target/s390x/kvm-stub.c | 10 +- >> target/s390x/kvm.c | 42 >> target/s390x/kvm_s390x.h | 4 +++- >> 4 files changed, 60 insertions(+), 12 deletions(-) >> >> diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c >> index 829ce6ad54..4973365d6c 100644 >> --- a/target/s390x/cpu.c >> +++ b/target/s390x/cpu.c >> @@ -139,8 +139,20 @@ static void s390_cpu_reset(CPUState *s, cpu_reset_type >> type) >> } >> >> /* Reset state inside the kernel that we cannot access yet from QEMU. */ > > For the last iteration, I asked about the 'yet' here... I have not written those comments, I merely refuse to delete them :) We still reset some state in the kernel, I'm not sure how much of that is already exposed via ioctls to QEMU, so I won't remove the comment. > >> -if (kvm_enabled() && type != S390_CPU_RESET_NORMAL) { >> -kvm_s390_reset_vcpu(cpu); >> +if (kvm_enabled()) { >> +switch (type) { >> +case S390_CPU_RESET_CLEAR: >> +kvm_s390_reset_vcpu_clear(cpu); >> +break; >> +case S390_CPU_RESET_INITIAL: >> +kvm_s390_reset_vcpu_initial(cpu); >> +break; >> +case S390_CPU_RESET_NORMAL: >> +kvm_s390_reset_vcpu_normal(cpu); >> +break; >> +default: >> +g_assert_not_reached(); >> +} >> } >> } >> > > (...) > >> @@ -403,17 +405,41 @@ int kvm_arch_destroy_vcpu(CPUState *cs) >> return 0; >> } >> >> -void kvm_s390_reset_vcpu(S390CPU *cpu) >> +static void kvm_s390_reset_vcpu(S390CPU *cpu, unsigned long type) >> { >> CPUState *cs = CPU(cpu); >> >> -/* The initial reset call is needed here to reset in-kernel >> - * vcpu data that we can't access directly from QEMU >> - * (i.e. with older kernels which don't support sync_regs/ONE_REG). >> - * Before this ioctl cpu_synchronize_state() is called in common kvm >> - * code (kvm-all) */ >> -if (kvm_vcpu_ioctl(cs, KVM_S390_INITIAL_RESET, NULL)) { >> -error_report("Initial CPU reset failed on CPU %i", cs->cpu_index); >> +/* >> + * The reset call is needed here to reset in-kernel vcpu data that >> + * we can't access directly from QEMU (i.e. with older kernels >> + * which don't support sync_regs/ONE_REG). Before this ioctl > > ...and this reference to 'older kernels' here. > > Are the comments still correct/relevant? See above > >> + * cpu_synchronize_state() is called in common kvm code >> + * (kvm-all). >> + */ >> +if (kvm_vcpu_ioctl(cs, type)) { >> +error_report("CPU reset failed on CPU %i type %lx", >> + cs->cpu_index, type); >> +} >> +} > signature.asc Description: OpenPGP digital signature
Re: [PATCH v3 2/4] s390x: Add missing vcpu reset functions
On Wed, 4 Dec 2019 10:00:45 +0100 Janosch Frank wrote: > On 12/3/19 6:44 PM, Cornelia Huck wrote: > > On Tue, 3 Dec 2019 08:28:11 -0500 > > Janosch Frank wrote: > > > >> Up to now we only had an ioctl to reset vcpu data QEMU couldn't reach > >> for the initial reset, and that was also called for the clear > >> reset. To be architecture compliant, we also need to clear local > >> interrupts on a normal reset. > >> > >> Because of this and the upcoming protvirt support we need to add > >> ioctls for the missing clear and normal resets. > >> > >> Signed-off-by: Janosch Frank > >> Reviewed-by: Thomas Huth > >> Acked-by: David Hildenbrand > >> --- > >> target/s390x/cpu.c | 16 +-- > >> target/s390x/kvm-stub.c | 10 +- > >> target/s390x/kvm.c | 42 > >> target/s390x/kvm_s390x.h | 4 +++- > >> 4 files changed, 60 insertions(+), 12 deletions(-) > >> > >> diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c > >> index 829ce6ad54..4973365d6c 100644 > >> --- a/target/s390x/cpu.c > >> +++ b/target/s390x/cpu.c > >> @@ -139,8 +139,20 @@ static void s390_cpu_reset(CPUState *s, > >> cpu_reset_type type) > >> } > >> > >> /* Reset state inside the kernel that we cannot access yet from QEMU. > >> */ > > > > For the last iteration, I asked about the 'yet' here... > > I have not written those comments, I merely refuse to delete them :) > We still reset some state in the kernel, I'm not sure how much of that > is already exposed via ioctls to QEMU, so I won't remove the comment. > > > > >> -if (kvm_enabled() && type != S390_CPU_RESET_NORMAL) { > >> -kvm_s390_reset_vcpu(cpu); > >> +if (kvm_enabled()) { > >> +switch (type) { > >> +case S390_CPU_RESET_CLEAR: > >> +kvm_s390_reset_vcpu_clear(cpu); > >> +break; > >> +case S390_CPU_RESET_INITIAL: > >> +kvm_s390_reset_vcpu_initial(cpu); > >> +break; > >> +case S390_CPU_RESET_NORMAL: > >> +kvm_s390_reset_vcpu_normal(cpu); > >> +break; > >> +default: > >> +g_assert_not_reached(); > >> +} > >> } > >> } > >> > > > > (...) > > > >> @@ -403,17 +405,41 @@ int kvm_arch_destroy_vcpu(CPUState *cs) > >> return 0; > >> } > >> > >> -void kvm_s390_reset_vcpu(S390CPU *cpu) > >> +static void kvm_s390_reset_vcpu(S390CPU *cpu, unsigned long type) > >> { > >> CPUState *cs = CPU(cpu); > >> > >> -/* The initial reset call is needed here to reset in-kernel > >> - * vcpu data that we can't access directly from QEMU > >> - * (i.e. with older kernels which don't support sync_regs/ONE_REG). > >> - * Before this ioctl cpu_synchronize_state() is called in common kvm > >> - * code (kvm-all) */ > >> -if (kvm_vcpu_ioctl(cs, KVM_S390_INITIAL_RESET, NULL)) { > >> -error_report("Initial CPU reset failed on CPU %i", cs->cpu_index); > >> +/* > >> + * The reset call is needed here to reset in-kernel vcpu data that > >> + * we can't access directly from QEMU (i.e. with older kernels > >> + * which don't support sync_regs/ONE_REG). Before this ioctl > > > > ...and this reference to 'older kernels' here. > > > > Are the comments still correct/relevant? > > See above Fair enough, let's keep them for now and revisit this later. > > > > >> + * cpu_synchronize_state() is called in common kvm code > >> + * (kvm-all). > >> + */ > >> +if (kvm_vcpu_ioctl(cs, type)) { > >> +error_report("CPU reset failed on CPU %i type %lx", > >> + cs->cpu_index, type); > >> +} > >> +} > > > > pgpHZKd3FGrTB.pgp Description: OpenPGP digital signature
Re: [PATCH v3 2/4] s390x: Add missing vcpu reset functions
On Tue, 3 Dec 2019 08:28:11 -0500 Janosch Frank wrote: > Up to now we only had an ioctl to reset vcpu data QEMU couldn't reach > for the initial reset, and that was also called for the clear > reset. To be architecture compliant, we also need to clear local > interrupts on a normal reset. Do we also need to do something like that for tcg? David? > > Because of this and the upcoming protvirt support we need to add > ioctls for the missing clear and normal resets. > > Signed-off-by: Janosch Frank > Reviewed-by: Thomas Huth > Acked-by: David Hildenbrand > --- > target/s390x/cpu.c | 16 +-- > target/s390x/kvm-stub.c | 10 +- > target/s390x/kvm.c | 42 > target/s390x/kvm_s390x.h | 4 +++- > 4 files changed, 60 insertions(+), 12 deletions(-)
Re: [PATCH v3 2/4] s390x: Add missing vcpu reset functions
On 04.12.19 15:59, Cornelia Huck wrote: > On Tue, 3 Dec 2019 08:28:11 -0500 > Janosch Frank wrote: > >> Up to now we only had an ioctl to reset vcpu data QEMU couldn't reach >> for the initial reset, and that was also called for the clear >> reset. To be architecture compliant, we also need to clear local >> interrupts on a normal reset. > > Do we also need to do something like that for tcg? David? > So, we have /* Fields up to this point are not cleared by initial CPU reset */ struct {} start_initial_reset_fields; [...] int pending_int uint16_t external_call_addr; DECLARE_BITMAP(emergency_signals, S390_MAX_CPUS); [...] /* Fields up to this point are cleared by a CPU reset */ struct {} end_reset_fields; This means, local interrupts will be cleared by everything that zeroes "start_initial_reset_fields->end_reset_fields" So, they will get cleared by S390_CPU_RESET_INITIAL only if I am not wrong. In order to clear them on S390_CPU_RESET_NORMAL, we have to manually set them to zero. (we really should wrap TCG-only fields by ifdefs) -- Thanks, David / dhildenb
Re: [PATCH v3 2/4] s390x: Add missing vcpu reset functions
On 04.12.19 16:07, David Hildenbrand wrote: > On 04.12.19 15:59, Cornelia Huck wrote: >> On Tue, 3 Dec 2019 08:28:11 -0500 >> Janosch Frank wrote: >> >>> Up to now we only had an ioctl to reset vcpu data QEMU couldn't reach >>> for the initial reset, and that was also called for the clear >>> reset. To be architecture compliant, we also need to clear local >>> interrupts on a normal reset. >> >> Do we also need to do something like that for tcg? David? >> > > So, we have > > /* Fields up to this point are not cleared by initial CPU reset */ > struct {} start_initial_reset_fields; > [...] > int pending_int > uint16_t external_call_addr; > DECLARE_BITMAP(emergency_signals, S390_MAX_CPUS); > [...] > /* Fields up to this point are cleared by a CPU reset */ > struct {} end_reset_fields; > > This means, local interrupts will be cleared by everything that zeroes > "start_initial_reset_fields->end_reset_fields" > > So, they will get cleared by S390_CPU_RESET_INITIAL only if I am not > wrong. In order to clear them on S390_CPU_RESET_NORMAL, we have to > manually set them to zero. Sorry, by S390_CPU_RESET_INITIAL and S390_CPU_RESET_CLEAR. > > (we really should wrap TCG-only fields by ifdefs) > -- Thanks, David / dhildenb
Re: [PATCH v3 2/4] s390x: Add missing vcpu reset functions
On Wed, 4 Dec 2019 16:08:48 +0100 David Hildenbrand wrote: > On 04.12.19 16:07, David Hildenbrand wrote: > > On 04.12.19 15:59, Cornelia Huck wrote: > >> On Tue, 3 Dec 2019 08:28:11 -0500 > >> Janosch Frank wrote: > >> > >>> Up to now we only had an ioctl to reset vcpu data QEMU couldn't reach > >>> for the initial reset, and that was also called for the clear > >>> reset. To be architecture compliant, we also need to clear local > >>> interrupts on a normal reset. > >> > >> Do we also need to do something like that for tcg? David? > >> > > > > So, we have > > > > /* Fields up to this point are not cleared by initial CPU reset */ > > struct {} start_initial_reset_fields; > > [...] > > int pending_int > > uint16_t external_call_addr; > > DECLARE_BITMAP(emergency_signals, S390_MAX_CPUS); > > [...] > > /* Fields up to this point are cleared by a CPU reset */ > > struct {} end_reset_fields; > > > > This means, local interrupts will be cleared by everything that zeroes > > "start_initial_reset_fields->end_reset_fields" > > > > So, they will get cleared by S390_CPU_RESET_INITIAL only if I am not > > wrong. In order to clear them on S390_CPU_RESET_NORMAL, we have to > > manually set them to zero. > > Sorry, by S390_CPU_RESET_INITIAL and S390_CPU_RESET_CLEAR. Ok. Will you cook up a patch, or should I do it? > > > > > (we really should wrap TCG-only fields by ifdefs) They probably do not get into the way, but making it obvious that they a tcg-only is something we really should do. > > > >
Re: [PATCH v3 2/4] s390x: Add missing vcpu reset functions
On 04.12.19 16:33, Cornelia Huck wrote: > On Wed, 4 Dec 2019 16:08:48 +0100 > David Hildenbrand wrote: > >> On 04.12.19 16:07, David Hildenbrand wrote: >>> On 04.12.19 15:59, Cornelia Huck wrote: On Tue, 3 Dec 2019 08:28:11 -0500 Janosch Frank wrote: > Up to now we only had an ioctl to reset vcpu data QEMU couldn't reach > for the initial reset, and that was also called for the clear > reset. To be architecture compliant, we also need to clear local > interrupts on a normal reset. Do we also need to do something like that for tcg? David? >>> >>> So, we have >>> >>> /* Fields up to this point are not cleared by initial CPU reset */ >>> struct {} start_initial_reset_fields; >>> [...] >>> int pending_int >>> uint16_t external_call_addr; >>> DECLARE_BITMAP(emergency_signals, S390_MAX_CPUS); >>> [...] >>> /* Fields up to this point are cleared by a CPU reset */ >>> struct {} end_reset_fields; >>> >>> This means, local interrupts will be cleared by everything that zeroes >>> "start_initial_reset_fields->end_reset_fields" >>> >>> So, they will get cleared by S390_CPU_RESET_INITIAL only if I am not >>> wrong. In order to clear them on S390_CPU_RESET_NORMAL, we have to >>> manually set them to zero. >> >> Sorry, by S390_CPU_RESET_INITIAL and S390_CPU_RESET_CLEAR. > > Ok. Will you cook up a patch, or should I do it? > If you want to have some TCG fun, feel free. Otherwise, let me know :) -- Thanks, David / dhildenb