From: Denis V. Lunev <[email protected]> Since Linux kernel commit de572491a975 ("KVM: x86: AMD's IBPB is not equivalent to Intel's IBPB") KVM offers spec-ctrl on AMD hosts only when the CPU enumerates IBPB_RET, which Zen 1-3 do not. A host-model guest started on such a CPU under an older kernel has spec-ctrl in its live definition. On a destination with the newer kernel QEMU filters the feature out and the incoming migration fails:
guest CPU doesn't match specification: missing features: spec-ctrl Restoring a save image or reverting a snapshot fails the same way, so the guest cannot be moved to the new kernel without a restart. Dropping the bit in libvirt costs the guest nothing beyond what KVM already withholds. KVM grants PRED_CMD for spec-ctrl or ibpb and SPEC_CTRL for spec-ctrl or any of ibrs, amd-stibp and amd-ssbd (kernel commit 39485ed95d6b), and checks the values written to SPEC_CTRL against the host, not the guest CPUID (kernel commit 841c2be09fe4). Both predate de572491a975. Since kernel commit e592ec657d84 KVM also ignores guest CPUID bits it does not support, so a host without IBPB_RET treats the guest as lacking spec-ctrl even if the bit stayed. QEMU drops the bit either way; libvirt only decides whether the guest survives. When a domain is started (fresh start, incoming migration, restore, snapshot revert) and the hypervisor dropped a required spec-ctrl while providing ibpb and one of ibrs, amd-stibp and amd-ssbd, virCPUUpdateLive() disables spec-ctrl in the live definition and logs it at VIR_INFO. spec-ctrl is also removed from the features QEMU reports as filtered, because check='full' fails on any filtered feature whatever the definition says. Any other missing or extra feature still fails with check='full'. Signed-off-by: Denis V. Lunev <[email protected]> --- src/cpu/cpu_x86.c | 93 ++++++++++++++++++- tests/cputest.c | 15 +++ ...PYC-7713-64-Core-no-spec-ctrl-disabled.xml | 4 + ...EPYC-7713-64-Core-no-spec-ctrl-enabled.xml | 11 +++ ...d-EPYC-7713-64-Core-no-spec-ctrl-guest.xml | 56 +++++++++++ ...id-EPYC-7713-64-Core-no-spec-ctrl-json.xml | 56 +++++++++++ ...-64-Core-no-spec-ctrl-no-ibpb-disabled.xml | 4 + ...3-64-Core-no-spec-ctrl-no-ibpb-enabled.xml | 11 +++ ...713-64-Core-no-spec-ctrl-no-ibpb-guest.xml | 56 +++++++++++ 9 files changed, 302 insertions(+), 4 deletions(-) create mode 100644 tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-disabled.xml create mode 100644 tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-enabled.xml create mode 100644 tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-guest.xml create mode 100644 tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-json.xml create mode 100644 tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-no-ibpb-disabled.xml create mode 100644 tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-no-ibpb-enabled.xml create mode 100644 tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-no-ibpb-guest.xml diff --git a/src/cpu/cpu_x86.c b/src/cpu/cpu_x86.c index 18c5756ef9..3e89d5ef0a 100644 --- a/src/cpu/cpu_x86.c +++ b/src/cpu/cpu_x86.c @@ -3144,6 +3144,88 @@ virCPUx86Update(virCPUDef *guest, } +/* + * Since kernel commit de572491a975 KVM no longer offers spec-ctrl on AMD + * hosts without IBPB_RET. The MSRs it stands for are still available to + * the guest through ibpb and any of ibrs, amd-stibp and amd-ssbd. + */ +static int +x86SpecCtrlDroppable(const virCPUDef *cpu G_GNUC_UNUSED, + virCPUx86Map *map, + const virCPUx86Data *enabled) +{ + const char *specCtrlMSR[] = { "ibrs", "amd-stibp", "amd-ssbd" }; + size_t i; + int rc; + + if ((rc = x86FeatureInData("ibpb", enabled, map)) <= 0) + return rc; + + for (i = 0; i < G_N_ELEMENTS(specCtrlMSR); i++) { + if ((rc = x86FeatureInData(specCtrlMSR[i], enabled, map)) != 0) + return rc; + } + + return 0; +} + + +static const struct { + const char *name; + int (*droppable)(const virCPUDef *cpu, + virCPUx86Map *map, + const virCPUx86Data *enabled); +} x86DroppableFeatures[] = { + { "spec-ctrl", x86SpecCtrlDroppable }, +}; + + +/* + * Newer KVM may stop offering a feature a guest started elsewhere already + * requires, although the guest loses nothing without it. Disable such + * features rather than failing the check. + */ +static int +x86UpdateLiveDroppable(virCPUDef *cpu, + virCPUx86Map *map, + const virCPUx86Data *enabled, + virCPUx86Data *disabled) +{ + size_t i; + + for (i = 0; i < G_N_ELEMENTS(x86DroppableFeatures); i++) { + const char *name = x86DroppableFeatures[i].name; + virCPUFeatureDef *feature; + virCPUx86Feature *x86Feature; + int rc; + + if (!(feature = virCPUDefFindFeature(cpu, name)) || + feature->policy != VIR_CPU_FEATURE_REQUIRE) + continue; + + if ((rc = x86FeatureInData(name, enabled, map)) < 0) + return -1; + + if (rc == 1) + continue; + + if ((rc = x86DroppableFeatures[i].droppable(cpu, map, enabled)) < 0) + return -1; + + if (rc == 0) + continue; + + VIR_INFO("Hypervisor does not provide '%s', disabling it in the guest CPU", + name); + virCPUDefUpdateFeature(cpu, name, VIR_CPU_FEATURE_DISABLE); + if ((x86Feature = x86FeatureFind(map, name))) + x86DataSubtract(disabled, &x86Feature->data); + } + + return 0; +} + + static int virCPUx86UpdateLive(virCPUDef *cpu, virCPUData *dataEnabled, @@ -3165,16 +3247,19 @@ virCPUx86UpdateLive(virCPUDef *cpu, if (!(map = virCPUx86GetMap())) return -1; - if (!(model = x86ModelFromCPU(cpu, map, -1)) || - !(modelDisabled = x86ModelFromCPU(cpu, map, VIR_CPU_FEATURE_DISABLE))) - return -1; - if (dataEnabled) x86DataCopy(&enabled, &dataEnabled->data.x86); if (dataDisabled) x86DataCopy(&disabled, &dataDisabled->data.x86); + if (x86UpdateLiveDroppable(cpu, map, &enabled, &disabled) < 0) + return -1; + + if (!(model = x86ModelFromCPU(cpu, map, -1)) || + !(modelDisabled = x86ModelFromCPU(cpu, map, VIR_CPU_FEATURE_DISABLE))) + return -1; + for (i = 0; i < map->nfeatures; i++) { virCPUx86Feature *feature = map->features[i]; virCPUFeaturePolicy expected = VIR_CPU_FEATURE_LAST; diff --git a/tests/cputest.c b/tests/cputest.c index e29c81cb8e..35b426155e 100644 --- a/tests/cputest.c +++ b/tests/cputest.c @@ -857,6 +857,15 @@ cpuTestUpdateLive(const void *arg) !(disabledData = virCPUDataParse(disabled))) return -1; + if (data->result == -1) { + if (virCPUUpdateLive(data->arch, cpu, enabledData, disabledData) == 0) { + VIR_TEST_VERBOSE("virCPUUpdateLive succeeded unexpectedly"); + return -1; + } + virResetLastError(); + return 0; + } + expectedFile = g_strdup_printf("cpuid-%s-json", data->host); if (!(expected = cpuTestLoadXML(data->arch, expectedFile, data->existingTestCases))) { @@ -1110,6 +1119,10 @@ mymain(void) } \ } while (0) +#define DO_TEST_UPDATE_LIVE(arch, host, result) \ + DO_TEST(arch, cpuTestUpdateLive, host, host, \ + NULL, NULL, 0, NULL, 0, result) + #define DO_TEST_CPUID_BASELINE(arch, label, cpu1, cpu2) \ do { \ const char *cpus[] = {cpu1, cpu2}; \ @@ -1270,6 +1283,8 @@ mymain(void) DO_TEST_CPUID(VIR_ARCH_X86_64, "EPYC-7601-32-Core-ibpb", JSON_MODELS_REQUIRED); DO_TEST_CPUID(VIR_ARCH_X86_64, "EPYC-7502-32-Core", JSON_MODELS); DO_TEST_CPUID(VIR_ARCH_X86_64, "EPYC-7713-64-Core", JSON_MODELS); + DO_TEST_UPDATE_LIVE(VIR_ARCH_X86_64, "EPYC-7713-64-Core-no-spec-ctrl", 0); + DO_TEST_UPDATE_LIVE(VIR_ARCH_X86_64, "EPYC-7713-64-Core-no-spec-ctrl-no-ibpb", -1); DO_TEST_CPUID(VIR_ARCH_X86_64, "EPYC-9334-32-Core", JSON_MODELS); DO_TEST_CPUID(VIR_ARCH_X86_64, "FX-8150", JSON_NONE); DO_TEST_CPUID(VIR_ARCH_X86_64, "Opteron-1352", JSON_NONE); diff --git a/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-disabled.xml b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-disabled.xml new file mode 100644 index 0000000000..d29d62bcad --- /dev/null +++ b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-disabled.xml @@ -0,0 +1,4 @@ +<!-- Features disabled by QEMU --> +<cpudata arch='x86'> + <cpuid eax_in='0x00000007' ecx_in='0x00' eax='0x00000000' ebx='0x00000000' ecx='0x00000000' edx='0x04000000'/> +</cpudata> diff --git a/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-enabled.xml b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-enabled.xml new file mode 100644 index 0000000000..6c7b14cd6f --- /dev/null +++ b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-enabled.xml @@ -0,0 +1,11 @@ +<!-- Features enabled by QEMU --> +<cpudata arch='x86'> + <cpuid eax_in='0x00000001' ecx_in='0x00' eax='0x00000000' ebx='0x00000000' ecx='0xf7fa3203' edx='0x078bfbff'/> + <cpuid eax_in='0x00000006' ecx_in='0x00' eax='0x00000004' ebx='0x00000000' ecx='0x00000000' edx='0x00000000'/> + <cpuid eax_in='0x00000007' ecx_in='0x00' eax='0x00000000' ebx='0x219c07ab' ecx='0x0040060c' edx='0xa8000010'/> + <cpuid eax_in='0x0000000d' ecx_in='0x01' eax='0x0000000f' ebx='0x00000000' ecx='0x00000000' edx='0x00000000'/> + <cpuid eax_in='0x80000001' ecx_in='0x00' eax='0x00000000' ebx='0x00000000' ecx='0x008003f7' edx='0x2e500800'/> + <cpuid eax_in='0x80000008' ecx_in='0x00' eax='0x00000000' ebx='0x03009205' ecx='0x00000000' edx='0x00000000'/> + <cpuid eax_in='0x8000000a' ecx_in='0x00' eax='0x00000000' ebx='0x00000000' ecx='0x00000000' edx='0x00000009'/> + <msr index='0x10a' edx='0x00000000' eax='0x00000069'/> +</cpudata> diff --git a/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-guest.xml b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-guest.xml new file mode 100644 index 0000000000..546896260f --- /dev/null +++ b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-guest.xml @@ -0,0 +1,56 @@ +<cpu mode='custom' match='exact' check='full'> + <model fallback='forbid'>EPYC-Milan</model> + <vendor>AMD</vendor> + <feature policy='require' name='x2apic'/> + <feature policy='require' name='tsc-deadline'/> + <feature policy='require' name='hypervisor'/> + <feature policy='require' name='tsc_adjust'/> + <feature policy='require' name='vaes'/> + <feature policy='require' name='vpclmulqdq'/> + <feature policy='require' name='spec-ctrl'/> + <feature policy='require' name='stibp'/> + <feature policy='require' name='arch-capabilities'/> + <feature policy='require' name='ssbd'/> + <feature policy='require' name='cmp_legacy'/> + <feature policy='require' name='virt-ssbd'/> + <feature policy='require' name='rdctl-no'/> + <feature policy='require' name='skip-l1dfl-vmentry'/> + <feature policy='require' name='mds-no'/> + <feature policy='require' name='pschange-mc-no'/> + <feature policy='disable' name='ibrs'/> + <feature policy='disable' name='svme-addr-chk'/> + <feature policy='disable' name='monitor'/> + <feature policy='disable' name='osxsave'/> + <feature policy='disable' name='ht'/> + <feature policy='disable' name='cmt'/> + <feature policy='disable' name='ospke'/> + <feature policy='disable' name='cet-ss'/> + <feature policy='disable' name='mbm_total'/> + <feature policy='disable' name='mbm_local'/> + <feature policy='disable' name='extapic'/> + <feature policy='disable' name='ibs'/> + <feature policy='disable' name='skinit'/> + <feature policy='disable' name='wdt'/> + <feature policy='disable' name='tce'/> + <feature policy='disable' name='topoext'/> + <feature policy='disable' name='perfctr_nb'/> + <feature policy='disable' name='overflow-recov'/> + <feature policy='disable' name='succor'/> + <feature policy='disable' name='invtsc'/> + <feature policy='disable' name='stibp-always-on'/> + <feature policy='disable' name='amd-psfd'/> + <feature policy='disable' name='lbrv'/> + <feature policy='disable' name='svm-lock'/> + <feature policy='disable' name='tsc-scale'/> + <feature policy='disable' name='vmcb-clean'/> + <feature policy='disable' name='flushbyasid'/> + <feature policy='disable' name='decodeassists'/> + <feature policy='disable' name='pause-filter'/> + <feature policy='disable' name='pfthreshold'/> + <feature policy='disable' name='v-vmsave-vmload'/> + <feature policy='disable' name='vgif'/> + <feature policy='disable' name='gmet'/> + <feature policy='disable' name='no-nested-data-bp'/> + <feature policy='disable' name='lfence-always-serializing'/> + <feature policy='disable' name='null-sel-clr-base'/> +</cpu> diff --git a/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-json.xml b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-json.xml new file mode 100644 index 0000000000..d68763d61b --- /dev/null +++ b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-json.xml @@ -0,0 +1,56 @@ +<cpu mode='custom' match='exact'> + <model fallback='forbid'>EPYC-Milan</model> + <vendor>AMD</vendor> + <feature policy='require' name='x2apic'/> + <feature policy='require' name='tsc-deadline'/> + <feature policy='require' name='hypervisor'/> + <feature policy='require' name='tsc_adjust'/> + <feature policy='require' name='vaes'/> + <feature policy='require' name='vpclmulqdq'/> + <feature policy='disable' name='spec-ctrl'/> + <feature policy='require' name='stibp'/> + <feature policy='require' name='arch-capabilities'/> + <feature policy='require' name='ssbd'/> + <feature policy='require' name='cmp_legacy'/> + <feature policy='require' name='virt-ssbd'/> + <feature policy='require' name='rdctl-no'/> + <feature policy='require' name='skip-l1dfl-vmentry'/> + <feature policy='require' name='mds-no'/> + <feature policy='require' name='pschange-mc-no'/> + <feature policy='disable' name='ibrs'/> + <feature policy='disable' name='svme-addr-chk'/> + <feature policy='disable' name='monitor'/> + <feature policy='disable' name='osxsave'/> + <feature policy='disable' name='ht'/> + <feature policy='disable' name='cmt'/> + <feature policy='disable' name='ospke'/> + <feature policy='disable' name='cet-ss'/> + <feature policy='disable' name='mbm_total'/> + <feature policy='disable' name='mbm_local'/> + <feature policy='disable' name='extapic'/> + <feature policy='disable' name='ibs'/> + <feature policy='disable' name='skinit'/> + <feature policy='disable' name='wdt'/> + <feature policy='disable' name='tce'/> + <feature policy='disable' name='topoext'/> + <feature policy='disable' name='perfctr_nb'/> + <feature policy='disable' name='overflow-recov'/> + <feature policy='disable' name='succor'/> + <feature policy='disable' name='invtsc'/> + <feature policy='disable' name='stibp-always-on'/> + <feature policy='disable' name='amd-psfd'/> + <feature policy='disable' name='lbrv'/> + <feature policy='disable' name='svm-lock'/> + <feature policy='disable' name='tsc-scale'/> + <feature policy='disable' name='vmcb-clean'/> + <feature policy='disable' name='flushbyasid'/> + <feature policy='disable' name='decodeassists'/> + <feature policy='disable' name='pause-filter'/> + <feature policy='disable' name='pfthreshold'/> + <feature policy='disable' name='v-vmsave-vmload'/> + <feature policy='disable' name='vgif'/> + <feature policy='disable' name='gmet'/> + <feature policy='disable' name='no-nested-data-bp'/> + <feature policy='disable' name='lfence-always-serializing'/> + <feature policy='disable' name='null-sel-clr-base'/> +</cpu> diff --git a/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-no-ibpb-disabled.xml b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-no-ibpb-disabled.xml new file mode 100644 index 0000000000..d29d62bcad --- /dev/null +++ b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-no-ibpb-disabled.xml @@ -0,0 +1,4 @@ +<!-- Features disabled by QEMU --> +<cpudata arch='x86'> + <cpuid eax_in='0x00000007' ecx_in='0x00' eax='0x00000000' ebx='0x00000000' ecx='0x00000000' edx='0x04000000'/> +</cpudata> diff --git a/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-no-ibpb-enabled.xml b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-no-ibpb-enabled.xml new file mode 100644 index 0000000000..0662433a08 --- /dev/null +++ b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-no-ibpb-enabled.xml @@ -0,0 +1,11 @@ +<!-- Features enabled by QEMU --> +<cpudata arch='x86'> + <cpuid eax_in='0x00000001' ecx_in='0x00' eax='0x00000000' ebx='0x00000000' ecx='0xf7fa3203' edx='0x078bfbff'/> + <cpuid eax_in='0x00000006' ecx_in='0x00' eax='0x00000004' ebx='0x00000000' ecx='0x00000000' edx='0x00000000'/> + <cpuid eax_in='0x00000007' ecx_in='0x00' eax='0x00000000' ebx='0x219c07ab' ecx='0x0040060c' edx='0xa8000010'/> + <cpuid eax_in='0x0000000d' ecx_in='0x01' eax='0x0000000f' ebx='0x00000000' ecx='0x00000000' edx='0x00000000'/> + <cpuid eax_in='0x80000001' ecx_in='0x00' eax='0x00000000' ebx='0x00000000' ecx='0x008003f7' edx='0x2e500800'/> + <cpuid eax_in='0x80000008' ecx_in='0x00' eax='0x00000000' ebx='0x03008205' ecx='0x00000000' edx='0x00000000'/> + <cpuid eax_in='0x8000000a' ecx_in='0x00' eax='0x00000000' ebx='0x00000000' ecx='0x00000000' edx='0x00000009'/> + <msr index='0x10a' edx='0x00000000' eax='0x00000069'/> +</cpudata> diff --git a/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-no-ibpb-guest.xml b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-no-ibpb-guest.xml new file mode 100644 index 0000000000..546896260f --- /dev/null +++ b/tests/cputestdata/x86_64-cpuid-EPYC-7713-64-Core-no-spec-ctrl-no-ibpb-guest.xml @@ -0,0 +1,56 @@ +<cpu mode='custom' match='exact' check='full'> + <model fallback='forbid'>EPYC-Milan</model> + <vendor>AMD</vendor> + <feature policy='require' name='x2apic'/> + <feature policy='require' name='tsc-deadline'/> + <feature policy='require' name='hypervisor'/> + <feature policy='require' name='tsc_adjust'/> + <feature policy='require' name='vaes'/> + <feature policy='require' name='vpclmulqdq'/> + <feature policy='require' name='spec-ctrl'/> + <feature policy='require' name='stibp'/> + <feature policy='require' name='arch-capabilities'/> + <feature policy='require' name='ssbd'/> + <feature policy='require' name='cmp_legacy'/> + <feature policy='require' name='virt-ssbd'/> + <feature policy='require' name='rdctl-no'/> + <feature policy='require' name='skip-l1dfl-vmentry'/> + <feature policy='require' name='mds-no'/> + <feature policy='require' name='pschange-mc-no'/> + <feature policy='disable' name='ibrs'/> + <feature policy='disable' name='svme-addr-chk'/> + <feature policy='disable' name='monitor'/> + <feature policy='disable' name='osxsave'/> + <feature policy='disable' name='ht'/> + <feature policy='disable' name='cmt'/> + <feature policy='disable' name='ospke'/> + <feature policy='disable' name='cet-ss'/> + <feature policy='disable' name='mbm_total'/> + <feature policy='disable' name='mbm_local'/> + <feature policy='disable' name='extapic'/> + <feature policy='disable' name='ibs'/> + <feature policy='disable' name='skinit'/> + <feature policy='disable' name='wdt'/> + <feature policy='disable' name='tce'/> + <feature policy='disable' name='topoext'/> + <feature policy='disable' name='perfctr_nb'/> + <feature policy='disable' name='overflow-recov'/> + <feature policy='disable' name='succor'/> + <feature policy='disable' name='invtsc'/> + <feature policy='disable' name='stibp-always-on'/> + <feature policy='disable' name='amd-psfd'/> + <feature policy='disable' name='lbrv'/> + <feature policy='disable' name='svm-lock'/> + <feature policy='disable' name='tsc-scale'/> + <feature policy='disable' name='vmcb-clean'/> + <feature policy='disable' name='flushbyasid'/> + <feature policy='disable' name='decodeassists'/> + <feature policy='disable' name='pause-filter'/> + <feature policy='disable' name='pfthreshold'/> + <feature policy='disable' name='v-vmsave-vmload'/> + <feature policy='disable' name='vgif'/> + <feature policy='disable' name='gmet'/> + <feature policy='disable' name='no-nested-data-bp'/> + <feature policy='disable' name='lfence-always-serializing'/> + <feature policy='disable' name='null-sel-clr-base'/> +</cpu> -- 2.53.0
