When dynamic memory unacceptance is active on AMD SEV-SNP, an accepted 4KB page may sit immediately adjacent to a 2MB block that has been unaccepted via PVALIDATE(validate=false). Speculative unaligned loads across the page boundary via load_unaligned_zeropad() (such as in dentry path walk) trigger a hardware #VC exception with error code SVM_EXIT_PAGE_NOT_VALIDATED (0x404) or SVM_EXIT_NPF (0x400) on the unvalidated neighbor page.
Define SVM_EXIT_PAGE_NOT_VALIDATED (0x404) and invoke fixup_exception() in vc_raw_handle_exception() when an SVM_EXIT_PAGE_NOT_VALIDATED or SVM_EXIT_NPF #VC exception occurs in kernel mode at an EX_TYPE_ZEROPAD instruction before dispatching to vc_handle_exitcode(), matching the existing EX_TYPE_ZEROPAD handling in the Intel TDX #VE handler (ve_raise_fault()). Signed-off-by: Kevin Loughlin <[email protected]> --- arch/x86/coco/sev/vc-handle.c | 29 +++++++++++++++++++++++++++++ arch/x86/include/uapi/asm/svm.h | 2 ++ 2 files changed, 31 insertions(+) diff --git a/arch/x86/coco/sev/vc-handle.c b/arch/x86/coco/sev/vc-handle.c index f25d302efceb..1ed00771d87d 100644 --- a/arch/x86/coco/sev/vc-handle.c +++ b/arch/x86/coco/sev/vc-handle.c @@ -18,6 +18,7 @@ #include <linux/io.h> #include <linux/psp-sev.h> #include <linux/efi.h> +#include <linux/extable.h> #include <uapi/linux/sev-guest.h> #include <asm/init.h> @@ -872,6 +873,29 @@ static __always_inline bool vc_from_invalid_context(struct pt_regs *regs) return is_vc2_stack(sp) && !is_vc2_stack(prev_sp); } +static bool vc_try_fixup_zeropad(struct es_em_ctxt *ctxt, struct pt_regs *regs, + unsigned long error_code) +{ + unsigned long addr, fault_addr; + + if ((error_code != SVM_EXIT_PAGE_NOT_VALIDATED && + error_code != SVM_EXIT_NPF) || + user_mode(regs) || + ex_get_fixup_type(regs->ip) != EX_TYPE_ZEROPAD) + return false; + + addr = (unsigned long)insn_get_addr_ref(&ctxt->insn, regs); + if (addr == ~0ul) + return false; + + /* + * load_unaligned_zeropad() faults when an unaligned word load crosses + * into an unaccepted page; round up to the crossed page boundary. + */ + fault_addr = (addr + sizeof(long) - 1) & PAGE_MASK; + return fixup_exception(regs, X86_TRAP_VC, error_code, fault_addr); +} + static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code) { struct ghcb_state state; @@ -885,6 +909,11 @@ static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_co vc_ghcb_invalidate(ghcb); result = vc_init_em_ctxt(&ctxt, regs, error_code); + if (result == ES_OK && vc_try_fixup_zeropad(&ctxt, regs, error_code)) { + __sev_put_ghcb(&state); + return true; + } + if (result == ES_OK) result = vc_handle_exitcode(&ctxt, ghcb, error_code); diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index e80676185092..38b8e0c2f5e2 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -101,6 +101,7 @@ #define SVM_EXIT_AVIC_INCOMPLETE_IPI 0x401 #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS 0x402 #define SVM_EXIT_VMGEXIT 0x403 +#define SVM_EXIT_PAGE_NOT_VALIDATED 0x404 #define SVM_EXIT_PML_FULL 0x407 /* SEV-ES software-defined VMGEXIT events */ @@ -237,6 +238,7 @@ { SVM_EXIT_AVIC_INCOMPLETE_IPI, "avic_incomplete_ipi" }, \ { SVM_EXIT_AVIC_UNACCELERATED_ACCESS, "avic_unaccelerated_access" }, \ { SVM_EXIT_VMGEXIT, "vmgexit" }, \ + { SVM_EXIT_PAGE_NOT_VALIDATED, "page_not_validated" }, \ { SVM_EXIT_PML_FULL, "pml_full" }, \ { SVM_VMGEXIT_MMIO_READ, "vmgexit_mmio_read" }, \ { SVM_VMGEXIT_MMIO_WRITE, "vmgexit_mmio_write" }, \ -- 2.56.0.307.g5fe7bd680c-goog
