Update the address of the allocated memory, only keep 4GB limitation for the case that APs still need to be transferred to 32-bit mode before OS.
Remove the unused arguments of AsmRelocateApLoopStart, update the stack offset. For the processors other than with SEV-ES enabled, keep APs in 64 bit mode before handoff to OS. Create PageTable for the allocated reserved memory. Tested on the OVMF package use of the support on AMD processors. Cc: Guo Dong <guo.d...@intel.com> Cc: Ray Ni <ray...@intel.com> Cc: Sean Rhodes <sean@starlabs.systems> Cc: James Lu <james...@intel.com> Cc: Gua Guo <gua....@intel.com> Cc: Gerd Hoffmann <kra...@redhat.com> Cc: Tom Lendacky <thomas.lenda...@amd.com> Cc: Laszlo Ersek <ler...@redhat.com> Tested-by: Tom Lendacky <thomas.lenda...@amd.com> Tested-by: Yuanhao Xie <yuanhao....@intel.com> Signed-off-by: Yuanhao Xie <yuanhao....@intel.com> --- UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf | 6 +++++- UefiCpuPkg/Library/MpInitLib/DxeMpLib.c | 28 ++++++++++++++++++---------- UefiCpuPkg/Library/MpInitLib/Ia32/CreatePageTable.c | 23 +++++++++++++++++++++++ UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm | 11 ++++------- UefiCpuPkg/Library/MpInitLib/MpEqu.inc | 22 +++++++++++----------- UefiCpuPkg/Library/MpInitLib/MpLib.h | 17 +++++++++++++---- UefiCpuPkg/Library/MpInitLib/X64/CreatePageTable.c | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm | 173 ++++++++++++++++++++++++++++------------------------------------------------------------------------------------------------------------------------------------------------- UefiCpuPkg/UefiCpuPkg.dsc | 3 ++- 9 files changed, 186 insertions(+), 179 deletions(-) diff --git a/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf b/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf index cd07de3a3c..4285dd06b4 100644 --- a/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf +++ b/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf @@ -1,7 +1,7 @@ ## @file # MP Initialize Library instance for DXE driver. # -# Copyright (c) 2016 - 2021, Intel Corporation. All rights reserved.<BR> +# Copyright (c) 2016 - 2023, Intel Corporation. All rights reserved.<BR> # SPDX-License-Identifier: BSD-2-Clause-Patent # ## @@ -24,10 +24,12 @@ [Sources.IA32] Ia32/AmdSev.c Ia32/MpFuncs.nasm + Ia32/CreatePageTable.c [Sources.X64] X64/AmdSev.c X64/MpFuncs.nasm + X64/CreatePageTable.c [Sources.common] AmdSev.c @@ -56,6 +58,8 @@ PcdLib CcExitLib MicrocodeLib +[LibraryClasses.X64] + CpuPageTableLib [Protocols] gEfiTimerArchProtocolGuid ## SOMETIMES_CONSUMES diff --git a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c index 75743faf5f..76953206ca 100644 --- a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c +++ b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c @@ -28,7 +28,7 @@ volatile BOOLEAN mStopCheckAllApsStatus = TRUE; UINTN mReservedTopOfApStack; volatile UINT32 mNumberToFinish = 0; RELOCATE_AP_LOOP_ENTRY mReservedApLoop; - +UINTN mApPageTable; // // Begin wakeup buffer allocation below 0x88000 @@ -379,10 +379,10 @@ RelocateApLoop ( IN OUT VOID *Buffer ) { - CPU_MP_DATA *CpuMpData; - BOOLEAN MwaitSupport; - UINTN ProcessorNumber; - UINTN StackStart; + CPU_MP_DATA *CpuMpData; + BOOLEAN MwaitSupport; + UINTN ProcessorNumber; + UINTN StackStart; MpInitLibWhoAmI (&ProcessorNumber); CpuMpData = GetCpuMpData (); @@ -404,12 +404,9 @@ RelocateApLoop ( mReservedApLoop.GenericEntry ( MwaitSupport, CpuMpData->ApTargetCState, - CpuMpData->PmCodeSegment, StackStart - ProcessorNumber * AP_SAFE_STACK_SIZE, (UINTN)&mNumberToFinish, - CpuMpData->Pm16CodeSegment, - CpuMpData->SevEsAPBuffer, - CpuMpData->WakeupBuffer + mApPageTable ); } @@ -540,9 +537,11 @@ InitMpGlobalData ( AddressMap = &CpuMpData->AddressMap; if (CpuMpData->UseSevEsAPMethod) { + Address = BASE_4GB - 1; ApLoopFunc = AddressMap->RelocateApLoopFuncAddressAmdSev; ApLoopFuncSize = AddressMap->RelocateApLoopFuncSizeAmdSev; } else { + Address = MAX_ADDRESS; ApLoopFunc = AddressMap->RelocateApLoopFuncAddress; ApLoopFuncSize = AddressMap->RelocateApLoopFuncSize; } @@ -564,7 +563,6 @@ InitMpGlobalData ( // +------------+ (low address ) // - Address = BASE_4GB - 1; StackPages = EFI_SIZE_TO_PAGES (CpuMpData->CpuCount * AP_SAFE_STACK_SIZE); FuncPages = EFI_SIZE_TO_PAGES (ALIGN_VALUE (ApLoopFuncSize, EFI_PAGE_SIZE)); @@ -597,6 +595,16 @@ InitMpGlobalData ( ASSERT ((mReservedTopOfApStack & (UINTN)(CPU_STACK_ALIGNMENT - 1)) == 0); mReservedApLoop.Data = (VOID *)(UINTN)Address; CopyMem (mReservedApLoop.Data, ApLoopFunc, ApLoopFuncSize); + if (!CpuMpData->UseSevEsAPMethod) { + // + // non-Sev Processor + // + mApPageTable = CreatePageTable ( + (UINTN)Address, + EFI_PAGES_TO_SIZE (StackPages+FuncPages) + ); + } + Status = gBS->CreateEvent ( EVT_TIMER | EVT_NOTIFY_SIGNAL, TPL_NOTIFY, diff --git a/UefiCpuPkg/Library/MpInitLib/Ia32/CreatePageTable.c b/UefiCpuPkg/Library/MpInitLib/Ia32/CreatePageTable.c new file mode 100644 index 0000000000..bec9b247c0 --- /dev/null +++ b/UefiCpuPkg/Library/MpInitLib/Ia32/CreatePageTable.c @@ -0,0 +1,23 @@ +/** @file + Function to create page talbe. + Only create page table for x64, and leave the CreatePageTable empty for Ia32. + Copyright (c) 2023, Intel Corporation. All rights reserved.<BR> + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include <Base.h> + +/** + Only create page table for x64, and leave the CreatePageTable empty for Ia32. + @param[in] LinearAddress The start of the linear address range. + @param[in] Length The length of the linear address range. + @return The page table to be created. +**/ +UINTN +CreatePageTable ( + IN UINTN Address, + IN UINTN Length + ) +{ + return 0; +} diff --git a/UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm b/UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm index bfcdbd31c1..c65a825a23 100644 --- a/UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm +++ b/UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm @@ -1,5 +1,5 @@ ;------------------------------------------------------------------------------ ; -; Copyright (c) 2015 - 2022, Intel Corporation. All rights reserved.<BR> +; Copyright (c) 2015 - 2023, Intel Corporation. All rights reserved.<BR> ; SPDX-License-Identifier: BSD-2-Clause-Patent ; ; Module Name: @@ -219,20 +219,17 @@ SwitchToRealProcEnd: RendezvousFunnelProcEnd: ;------------------------------------------------------------------------------------- -; AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer); -; -; The last three parameters (Pm16CodeSegment, SevEsAPJumpTable and WakeupBuffer) are -; specific to SEV-ES support and are not applicable on IA32. +; AsmRelocateApLoop (MwaitSupport, ApTargetCState, TopOfApStack, CountTofinish, Cr3); ;------------------------------------------------------------------------------------- AsmRelocateApLoopStart: mov eax, esp - mov esp, [eax + 16] ; TopOfApStack + mov esp, [eax + 12] ; TopOfApStack push dword [eax] ; push return address for stack trace push ebp mov ebp, esp mov ebx, [eax + 8] ; ApTargetCState mov ecx, [eax + 4] ; MwaitSupport - mov eax, [eax + 20] ; CountTofinish + mov eax, [eax + 16] ; CountTofinish lock dec dword [eax] ; (*CountTofinish)-- cmp cl, 1 ; Check mwait-monitor support jnz HltLoop diff --git a/UefiCpuPkg/Library/MpInitLib/MpEqu.inc b/UefiCpuPkg/Library/MpInitLib/MpEqu.inc index 1472ef2024..6730f2f411 100644 --- a/UefiCpuPkg/Library/MpInitLib/MpEqu.inc +++ b/UefiCpuPkg/Library/MpInitLib/MpEqu.inc @@ -21,17 +21,17 @@ CPU_SWITCH_STATE_LOADED equ 2 ; Equivalent NASM structure of MP_ASSEMBLY_ADDRESS_MAP ; struc MP_ASSEMBLY_ADDRESS_MAP - .RendezvousFunnelAddress CTYPE_UINTN 1 - .ModeEntryOffset CTYPE_UINTN 1 - .RendezvousFunnelSize CTYPE_UINTN 1 - .RelocateApLoopFuncAddress CTYPE_UINTN 1 - .RelocateApLoopFuncSize CTYPE_UINTN 1 - .RelocateApLoopFuncAddressAmdSev CTYPE_UINTN 1 - .RelocateApLoopFuncSizeAmdSev CTYPE_UINTN 1 - .ModeTransitionOffset CTYPE_UINTN 1 - .SwitchToRealNoNxOffset CTYPE_UINTN 1 - .SwitchToRealPM16ModeOffset CTYPE_UINTN 1 - .SwitchToRealPM16ModeSize CTYPE_UINTN 1 + .RendezvousFunnelAddress CTYPE_UINTN 1 + .ModeEntryOffset CTYPE_UINTN 1 + .RendezvousFunnelSize CTYPE_UINTN 1 + .RelocateApLoopFuncAddress CTYPE_UINTN 1 + .RelocateApLoopFuncSize CTYPE_UINTN 1 + .RelocateApLoopFuncAddressAmdSev CTYPE_UINTN 1 + .RelocateApLoopFuncSizeAmdSev CTYPE_UINTN 1 + .ModeTransitionOffset CTYPE_UINTN 1 + .SwitchToRealNoNxOffset CTYPE_UINTN 1 + .SwitchToRealPM16ModeOffset CTYPE_UINTN 1 + .SwitchToRealPM16ModeSize CTYPE_UINTN 1 endstruc ; diff --git a/UefiCpuPkg/Library/MpInitLib/MpLib.h b/UefiCpuPkg/Library/MpInitLib/MpLib.h index 772a828045..cc77843bea 100644 --- a/UefiCpuPkg/Library/MpInitLib/MpLib.h +++ b/UefiCpuPkg/Library/MpInitLib/MpLib.h @@ -368,12 +368,9 @@ typedef (EFIAPI *ASM_RELOCATE_AP_LOOP)( IN BOOLEAN MwaitSupport, IN UINTN ApTargetCState, - IN UINTN PmCodeSegment, IN UINTN TopOfApStack, IN UINTN NumberToFinish, - IN UINTN Pm16CodeSegment, - IN UINTN SevEsAPJumpTable, - IN UINTN WakeupBuffer + IN UINTN Cr3 ); /** @@ -498,6 +495,18 @@ GetSevEsAPMemory ( VOID ); +/** + Create 1:1 mapping page table in reserved memory to map the specified address range. + @param[in] LinearAddress The start of the linear address range. + @param[in] Length The length of the linear address range. + @return The page table to be created. +**/ +UINTN +CreatePageTable ( + IN UINTN Address, + IN UINTN Length + ); + /** This function will be called by BSP to wakeup AP. diff --git a/UefiCpuPkg/Library/MpInitLib/X64/CreatePageTable.c b/UefiCpuPkg/Library/MpInitLib/X64/CreatePageTable.c new file mode 100644 index 0000000000..7cf91ed9c4 --- /dev/null +++ b/UefiCpuPkg/Library/MpInitLib/X64/CreatePageTable.c @@ -0,0 +1,82 @@ +/** @file + Function to create page talbe. + Only create page table for x64, and leave the CreatePageTable empty for Ia32. + Copyright (c) 2023, Intel Corporation. All rights reserved.<BR> + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ +#include <Library/CpuPageTableLib.h> +#include <Library/MemoryAllocationLib.h> +#include <Base.h> +#include <Library/BaseMemoryLib.h> +#include <Library/DebugLib.h> +#include <Library/BaseLib.h> + +/** + Create 1:1 mapping page table in reserved memory to map the specified address range. + @param[in] LinearAddress The start of the linear address range. + @param[in] Length The length of the linear address range. + @return The page table to be created. +**/ +UINTN +CreatePageTable ( + IN UINTN Address, + IN UINTN Length + ) +{ + EFI_STATUS Status; + VOID *PageTableBuffer; + UINTN PageTableBufferSize; + UINTN PageTable; + PAGING_MODE PagingMode; + IA32_CR4 Cr4; + + IA32_MAP_ATTRIBUTE MapAttribute; + IA32_MAP_ATTRIBUTE MapMask; + + MapAttribute.Uint64 = Address; + MapAttribute.Bits.Present = 1; + MapAttribute.Bits.ReadWrite = 1; + + MapMask.Bits.PageTableBaseAddress = 1; + MapMask.Bits.Present = 1; + MapMask.Bits.ReadWrite = 1; + + PageTable = 0; + PageTableBufferSize = 0; + + Cr4.UintN = AsmReadCr4 (); + + if (Cr4.Bits.LA57 == 1) { + PagingMode = Paging5Level; + } else { + PagingMode = Paging4Level; + } + + Status = PageTableMap ( + &PageTable, + PagingMode, + NULL, + &PageTableBufferSize, + Address, + Length, + &MapAttribute, + &MapMask + ); + ASSERT (Status == EFI_BUFFER_TOO_SMALL); + DEBUG ((DEBUG_INFO, "AP Page Table Buffer Size = %x\n", PageTableBufferSize)); + + PageTableBuffer = AllocateReservedPages (EFI_SIZE_TO_PAGES (PageTableBufferSize)); + ASSERT (PageTableBuffer != NULL); + Status = PageTableMap ( + &PageTable, + PagingMode, + PageTableBuffer, + &PageTableBufferSize, + Address, + Length, + &MapAttribute, + &MapMask + ); + ASSERT_EFI_ERROR (Status); + return PageTable; +} diff --git a/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm b/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm index d36f8ba06d..2bce04d99c 100644 --- a/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm +++ b/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm @@ -279,172 +279,55 @@ CProcedureInvoke: RendezvousFunnelProcEnd: ;------------------------------------------------------------------------------------- -; AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer); +; AsmRelocateApLoop (MwaitSupport, ApTargetCState, TopOfApStack, CountTofinish, Cr3); +; This function is called during the finalizaiton of Mp initialization before booting +; to OS, and aim to put Aps either in Mwait or HLT. ;------------------------------------------------------------------------------------- -AsmRelocateApLoopStart: -BITS 64 - cmp qword [rsp + 56], 0 ; SevEsAPJumpTable - je NoSevEs - - ; - ; Perform some SEV-ES related setup before leaving 64-bit mode - ; - push rcx - push rdx - - ; - ; Get the RDX reset value using CPUID - ; - mov rax, 1 - cpuid - mov rsi, rax ; Save off the reset value for RDX - - ; - ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call - ; - Must be done while in 64-bit long mode so that writes to - ; the GHCB memory will be unencrypted. - ; - No NAE events can be generated once this is set otherwise - ; the AP_RESET_HOLD SW_EXITCODE will be overwritten. - ; - mov rcx, 0xc0010130 - rdmsr ; Retrieve current GHCB address - shl rdx, 32 - or rdx, rax - - mov rdi, rdx - xor rax, rax - mov rcx, 0x800 - shr rcx, 3 - rep stosq ; Clear the GHCB - - mov rax, 0x80000004 ; VMGEXIT AP_RESET_HOLD - mov [rdx + 0x390], rax - mov rax, 114 ; Set SwExitCode valid bit - bts [rdx + 0x3f0], rax - inc rax ; Set SwExitInfo1 valid bit - bts [rdx + 0x3f0], rax - inc rax ; Set SwExitInfo2 valid bit - bts [rdx + 0x3f0], rax +; +----------------+ +; | Cr3 | rsp+40 +; +----------------+ +; | CountTofinish | r9 +; +----------------+ +; | TopOfApStack | r8 +; +----------------+ +; | ApTargetCState | rdx +; +----------------+ +; | MwaitSupport | rcx +; +----------------+ +; | the return | +; +----------------+ low address - pop rdx - pop rcx - -NoSevEs: - cli ; Disable interrupt before switching to 32-bit mode - mov rax, [rsp + 40] ; CountTofinish +AsmRelocateApLoopStart: + mov rax, r9 ; CountTofinish lock dec dword [rax] ; (*CountTofinish)-- - mov r10, [rsp + 48] ; Pm16CodeSegment - mov rax, [rsp + 56] ; SevEsAPJumpTable - mov rbx, [rsp + 64] ; WakeupBuffer - mov rsp, r9 ; TopOfApStack - - push rax ; Save SevEsAPJumpTable - push rbx ; Save WakeupBuffer - push r10 ; Save Pm16CodeSegment - push rcx ; Save MwaitSupport - push rdx ; Save ApTargetCState - - lea rax, [PmEntry] ; rax <- The start address of transition code - - push r8 - push rax - - ; - ; Clear R8 - R15, for reset, before going into 32-bit mode - ; - xor r8, r8 - xor r9, r9 - xor r10, r10 - xor r11, r11 - xor r12, r12 - xor r13, r13 - xor r14, r14 - xor r15, r15 - - ; - ; Far return into 32-bit mode - ; - retfq - -BITS 32 -PmEntry: - mov eax, cr0 - btr eax, 31 ; Clear CR0.PG - mov cr0, eax ; Disable paging and caches - - mov ecx, 0xc0000080 - rdmsr - and ah, ~ 1 ; Clear LME - wrmsr - mov eax, cr4 - and al, ~ (1 << 5) ; Clear PAE - mov cr4, eax - - pop edx - add esp, 4 - pop ecx, - add esp, 4 + mov rax, [rsp + 40] ; Cr3 + ; Do not push on old stack, since old stack is not mapped + ; in the page table pointed by cr3 + mov cr3, rax + mov rsp, r8 ; TopOfApStack MwaitCheck: cmp cl, 1 ; Check mwait-monitor support jnz HltLoop - mov ebx, edx ; Save C-State to ebx + mov rbx, rdx ; Save C-State to ebx + MwaitLoop: cli - mov eax, esp ; Set Monitor Address + mov rax, rsp ; Set Monitor Address xor ecx, ecx ; ecx = 0 xor edx, edx ; edx = 0 monitor - mov eax, ebx ; Mwait Cx, Target C-State per eax[7:4] + mov rax, rbx ; Mwait Cx, Target C-State per eax[7:4] shl eax, 4 mwait jmp MwaitLoop HltLoop: - pop edx ; PM16CodeSegment - add esp, 4 - pop ebx ; WakeupBuffer - add esp, 4 - pop eax ; SevEsAPJumpTable - add esp, 4 - cmp eax, 0 ; Check for SEV-ES - je DoHlt - - cli - ; - ; SEV-ES is enabled, use VMGEXIT (GHCB information already - ; set by caller) - ; -BITS 64 - rep vmmcall -BITS 32 - - ; - ; Back from VMGEXIT AP_HLT_LOOP - ; Push the FLAGS/CS/IP values to use - ; - push word 0x0002 ; EFLAGS - xor ecx, ecx - mov cx, [eax + 2] ; CS - push cx - mov cx, [eax] ; IP - push cx - push word 0x0000 ; For alignment, will be discarded - - push edx - push ebx - - mov edx, esi ; Restore RDX reset value - - retf - -DoHlt: cli hlt - jmp DoHlt + jmp HltLoop -BITS 64 AsmRelocateApLoopEnd: ;------------------------------------------------------------------------------------- diff --git a/UefiCpuPkg/UefiCpuPkg.dsc b/UefiCpuPkg/UefiCpuPkg.dsc index f9a46089d2..062a43d7a7 100644 --- a/UefiCpuPkg/UefiCpuPkg.dsc +++ b/UefiCpuPkg/UefiCpuPkg.dsc @@ -1,7 +1,7 @@ ## @file # UefiCpuPkg Package # -# Copyright (c) 2007 - 2022, Intel Corporation. All rights reserved.<BR> +# Copyright (c) 2007 - 2023, Intel Corporation. All rights reserved.<BR> # # SPDX-License-Identifier: BSD-2-Clause-Patent # @@ -94,6 +94,7 @@ MemoryAllocationLib|MdePkg/Library/UefiMemoryAllocationLib/UefiMemoryAllocationLib.inf HobLib|MdePkg/Library/DxeHobLib/DxeHobLib.inf CpuExceptionHandlerLib|UefiCpuPkg/Library/CpuExceptionHandlerLib/DxeCpuExceptionHandlerLib.inf + CpuPageTableLib|UefiCpuPkg/Library/CpuPageTableLib/CpuPageTableLib.inf MpInitLib|UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf RegisterCpuFeaturesLib|UefiCpuPkg/Library/RegisterCpuFeaturesLib/DxeRegisterCpuFeaturesLib.inf CpuCacheInfoLib|UefiCpuPkg/Library/CpuCacheInfoLib/DxeCpuCacheInfoLib.inf -- 2.36.1.windows.1 -=-=-=-=-=-=-=-=-=-=-=- Groups.io Links: You receive all messages sent to this group. View/Reply Online (#100358): https://edk2.groups.io/g/devel/message/100358 Mute This Topic: https://groups.io/mt/97081033/21656 Group Owner: devel+ow...@edk2.groups.io Unsubscribe: https://edk2.groups.io/g/devel/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-