https://github.com/jroelofs updated https://github.com/llvm/llvm-project/pull/73686
>From bc152095691b32d1ad8539dfd60f5089df5eed8d Mon Sep 17 00:00:00 2001 From: Jon Roelofs <jonathan_roel...@apple.com> Date: Tue, 28 Nov 2023 10:39:44 -0800 Subject: [PATCH 1/9] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?= =?UTF-8?q?itial=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Created using spr 1.3.4 --- llvm/docs/LangRef.rst | 7 +- llvm/include/llvm/CodeGen/AsmPrinter.h | 6 +- llvm/lib/CodeGen/GlobalISel/CallLowering.cpp | 7 +- llvm/lib/IR/Verifier.cpp | 12 +- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 308 ++++++++++++++++++ llvm/lib/Target/X86/X86AsmPrinter.cpp | 28 ++ llvm/lib/Target/X86/X86AsmPrinter.h | 1 + .../AArch64/GlobalISel/call-lowering-ifunc.ll | 37 +++ llvm/test/CodeGen/AArch64/addrsig-macho.ll | 4 +- llvm/test/CodeGen/AArch64/ifunc-asm.ll | 82 +++++ llvm/test/CodeGen/X86/ifunc-asm.ll | 28 +- llvm/test/Verifier/ifunc-macho.ll | 42 +++ 12 files changed, 539 insertions(+), 23 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-ifunc.ll create mode 100644 llvm/test/CodeGen/AArch64/ifunc-asm.ll create mode 100644 llvm/test/Verifier/ifunc-macho.ll diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index e448c5ed5c5d9..cb222e979db29 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -934,10 +934,11 @@ IFuncs ------- IFuncs, like as aliases, don't create any new data or func. They are just a new -symbol that dynamic linker resolves at runtime by calling a resolver function. +symbol that is resolved at runtime by calling a resolver function. -IFuncs have a name and a resolver that is a function called by dynamic linker -that returns address of another function associated with the name. +On ELF platforms, IFuncs are resolved by the dynamic linker at load time. On +MachO platforms, they are lowered in terms of ``.symbol_resolver``s, which +lazily resolve the callee the first time they are called. IFunc may have an optional :ref:`linkage type <linkage>` and an optional :ref:`visibility style <visibility>`. diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h index 2731ef452c79c..48fa6c478464c 100644 --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -882,7 +882,11 @@ class AsmPrinter : public MachineFunctionPass { GCMetadataPrinter *getOrCreateGCPrinter(GCStrategy &S); void emitGlobalAlias(Module &M, const GlobalAlias &GA); - void emitGlobalIFunc(Module &M, const GlobalIFunc &GI); + +protected: + virtual void emitGlobalIFunc(Module &M, const GlobalIFunc &GI); + +private: /// This method decides whether the specified basic block requires a label. bool shouldEmitLabelForBasicBlock(const MachineBasicBlock &MBB) const; diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp index 2527b14312896..e0080b145d4f9 100644 --- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -144,7 +144,12 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, // Try looking through a bitcast from one function type to another. // Commonly happens with calls to objc_msgSend(). const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts(); - if (const Function *F = dyn_cast<Function>(CalleeV)) + if (const GlobalIFunc *IF = dyn_cast<GlobalIFunc>(CalleeV); + IF && MF.getTarget().getTargetTriple().isOSBinFormatMachO()) { + // ld64 requires that .symbol_resolvers to be called via a stub, so these + // must always be a diret call. + Info.Callee = MachineOperand::CreateGA(IF, 0); + } else if (const Function *F = dyn_cast<Function>(CalleeV)) Info.Callee = MachineOperand::CreateGA(F, 0); else Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false); diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 5560c037aa3ee..94e76a43bf38d 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -959,6 +959,7 @@ void Verifier::visitGlobalIFunc(const GlobalIFunc &GI) { GlobalIFunc::getResolverFunctionType(GI.getValueType()); Check(ResolverTy == ResolverFuncTy->getPointerTo(GI.getAddressSpace()), "IFunc resolver has incorrect type", &GI); + } void Verifier::visitNamedMDNode(const NamedMDNode &NMD) { @@ -2239,13 +2240,10 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, } // Check EVEX512 feature. - if (MaxParameterWidth >= 512 && Attrs.hasFnAttr("target-features")) { - Triple T(M.getTargetTriple()); - if (T.isX86()) { - StringRef TF = Attrs.getFnAttr("target-features").getValueAsString(); - Check(!TF.contains("+avx512f") || !TF.contains("-evex512"), - "512-bit vector arguments require 'evex512' for AVX512", V); - } + if (MaxParameterWidth >= 512 && Attrs.hasFnAttr("target-features") && TT.isX86()) { + StringRef TF = Attrs.getFnAttr("target-features").getValueAsString(); + Check(!TF.contains("+avx512f") || !TF.contains("-evex512"), + "512-bit vector arguments require 'evex512' for AVX512", V); } checkUnsignedBaseTenFuncAttr(Attrs, "patchable-function-prefix", V); diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index f4d3a85f34c88..2dab8e126c9ab 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -30,6 +30,7 @@ #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/BinaryFormat/MachO.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/FaultMaps.h" #include "llvm/CodeGen/MachineBasicBlock.h" @@ -47,10 +48,12 @@ #include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstBuilder.h" #include "llvm/MC/MCSectionELF.h" +#include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" @@ -68,6 +71,22 @@ using namespace llvm; namespace { +enum class IFuncLowering { SymbolResolverIfSupported, SymbolResolverAlways, SymbolResolverNever }; + +static cl::opt<IFuncLowering> PreferredIFuncLowering( + "arm64-darwin-ifunc-symbol_resolver", cl::init(IFuncLowering::SymbolResolverNever), + cl::desc("Pick the lowering for ifuncs on darwin platforms"), cl::Hidden, + cl::values( + clEnumValN( + IFuncLowering::SymbolResolverIfSupported, "if_supported", + "Use .symbol_resolver's when known to be supported by the linker."), + clEnumValN(IFuncLowering::SymbolResolverAlways, "always", + "Always use .symbol_resolvers. NOTE: this might not be " + "supported by the linker in all cases."), + clEnumValN(IFuncLowering::SymbolResolverNever, "never", + "Use a manual lowering, doing what the linker would have " + "done, but in the compiler."))); + class AArch64AsmPrinter : public AsmPrinter { AArch64MCInstLower MCInstLowering; FaultMaps FM; @@ -198,6 +217,11 @@ class AArch64AsmPrinter : public AsmPrinter { bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override { return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags; } + + void emitGlobalIFunc(Module &M, const GlobalIFunc &GI) override; + + void emitLinkerSymbolResolver(Module &M, const GlobalIFunc &GI); + void emitManualSymbolResolver(Module &M, const GlobalIFunc &GI); }; } // end anonymous namespace @@ -1809,6 +1833,290 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, TmpInst); } +void AArch64AsmPrinter::emitLinkerSymbolResolver(Module &M, + const GlobalIFunc &GI) { + OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection()); + + MCSymbol *Name = getSymbol(&GI); + + // NOTE: non-global .symbol_resolvers are not yet supported by Darwin linkers + + if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) + OutStreamer->emitSymbolAttribute(Name, MCSA_Global); + else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) + OutStreamer->emitSymbolAttribute(Name, MCSA_WeakReference); + else + assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); + + OutStreamer->emitCodeAlignment(Align(4), STI); + OutStreamer->emitLabel(Name); + OutStreamer->emitSymbolAttribute(Name, MCSA_SymbolResolver); + emitVisibility(Name, GI.getVisibility()); + + // ld-prime does not seem to support aliases of symbol resolvers, so we have to + // tail call the resolver manually. + OutStreamer->emitInstruction( + MCInstBuilder(AArch64::B) + .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), + *STI); +} + +/// \brief Emit a manually-constructed .symbol_resolver that implements the +/// symbol resolution duties of the IFunc. +/// +/// Normally, this would be handled by linker magic, but unfortunately there are +/// a few limitations in ld64 and ld-prime's implementation of .symbol_resolver +/// that mean we can't always use them: +/// +/// * resolvers cannot be the target of an alias +/// * resolvers cannot have private linkage +/// * resolvers cannot have linkonce linkage +/// * resolvers cannot appear in executables +/// * resolvers cannot appear in bundles +/// +/// This works around that by emitting a close approximation of what the linker +/// would have done. +void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, + const GlobalIFunc &GI) { + auto EmitLinkage = [&](MCSymbol *Sym) { + if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) + OutStreamer->emitSymbolAttribute(Sym, MCSA_Global); + else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) + OutStreamer->emitSymbolAttribute(Sym, MCSA_WeakReference); + else + assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); + }; + + MCSymbol *LazyPointer = TM.getObjFileLowering()->getContext().getOrCreateSymbol( + "_" + GI.getName() + ".lazy_pointer"); + MCSymbol *StubHelper = + TM.getObjFileLowering()->getContext().getOrCreateSymbol( + "_" + GI.getName() + ".stub_helper"); + + OutStreamer->switchSection(OutContext.getObjectFileInfo()->getDataSection()); + + EmitLinkage(LazyPointer); + OutStreamer->emitLabel(LazyPointer); + emitVisibility(LazyPointer, GI.getVisibility()); + OutStreamer->emitValue(MCSymbolRefExpr::create(StubHelper, OutContext), 8); + + OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection()); + + MCSymbol *Stub = getSymbol(&GI); + + EmitLinkage(Stub); + OutStreamer->emitCodeAlignment(Align(4), STI); + OutStreamer->emitLabel(Stub); + emitVisibility(Stub, GI.getVisibility()); + + // adrp x16, lazy_pointer@GOTPAGE + // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] + // ldr x16, [x16] + // br x16 + + { + MCInst Adrp; + Adrp.setOpcode(AArch64::ADRP); + Adrp.addOperand(MCOperand::createReg(AArch64::X16)); + MCOperand SymPage; + MCInstLowering.lowerOperand( + MachineOperand::CreateES(LazyPointer->getName().data() + 1, + AArch64II::MO_GOT | AArch64II::MO_PAGE), + SymPage); + Adrp.addOperand(SymPage); + OutStreamer->emitInstruction(Adrp, *STI); + } + + { + MCInst Ldr; + Ldr.setOpcode(AArch64::LDRXui); + Ldr.addOperand(MCOperand::createReg(AArch64::X16)); + Ldr.addOperand(MCOperand::createReg(AArch64::X16)); + MCOperand SymPageOff; + MCInstLowering.lowerOperand( + MachineOperand::CreateES(LazyPointer->getName().data() + 1, + AArch64II::MO_GOT | AArch64II::MO_PAGEOFF), + SymPageOff); + Ldr.addOperand(SymPageOff); + Ldr.addOperand(MCOperand::createImm(0)); + OutStreamer->emitInstruction(Ldr, *STI); + } + + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRXui) + .addReg(AArch64::X16) + .addReg(AArch64::X16) + .addImm(0), *STI); + + OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() + ? AArch64::BRAAZ + : AArch64::BR) + .addReg(AArch64::X16), + *STI); + + EmitLinkage(StubHelper); + OutStreamer->emitCodeAlignment(Align(4), STI); + OutStreamer->emitLabel(StubHelper); + emitVisibility(StubHelper, GI.getVisibility()); + + // stp fp, lr, [sp, #-16]! + // mov fp, sp + // stp x1, x0, [sp, #-16]! + // stp x3, x2, [sp, #-16]! + // stp x5, x4, [sp, #-16]! + // stp x7, x6, [sp, #-16]! + // stp d1, d0, [sp, #-16]! + // stp d3, d2, [sp, #-16]! + // stp d5, d4, [sp, #-16]! + // stp d7, d6, [sp, #-16]! + // bl _resolver + // adrp x16, lazy_pointer@GOTPAGE + // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] + // str x0, [x16] + // mov x16, x0 + // ldp d7, d6, [sp], #16 + // ldp d5, d4, [sp], #16 + // ldp d3, d2, [sp], #16 + // ldp d1, d0, [sp], #16 + // ldp x7, x6, [sp], #16 + // ldp x5, x4, [sp], #16 + // ldp x3, x2, [sp], #16 + // ldp x1, x0, [sp], #16 + // ldp fp, lr, [sp], #16 + // br x16 + + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) + .addReg(AArch64::SP) + .addReg(AArch64::FP) + .addReg(AArch64::LR) + .addReg(AArch64::SP) + .addImm(-2), + *STI); + + OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) + .addReg(AArch64::FP) + .addReg(AArch64::SP) + .addImm(0) + .addImm(0), + *STI); + + for (int I = 0; I != 4; ++I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) + .addReg(AArch64::SP) + .addReg(AArch64::X1 + 2 * I) + .addReg(AArch64::X0 + 2 * I) + .addReg(AArch64::SP) + .addImm(-2), + *STI); + + for (int I = 0; I != 4; ++I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre) + .addReg(AArch64::SP) + .addReg(AArch64::D1 + 2 * I) + .addReg(AArch64::D0 + 2 * I) + .addReg(AArch64::SP) + .addImm(-2), + *STI); + + OutStreamer->emitInstruction( + MCInstBuilder(AArch64::BL) + .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), + *STI); + + { + MCInst Adrp; + Adrp.setOpcode(AArch64::ADRP); + Adrp.addOperand(MCOperand::createReg(AArch64::X16)); + MCOperand SymPage; + MCInstLowering.lowerOperand( + MachineOperand::CreateES(LazyPointer->getName().data() + 1, + AArch64II::MO_GOT | AArch64II::MO_PAGE), + SymPage); + Adrp.addOperand(SymPage); + OutStreamer->emitInstruction(Adrp, *STI); + } + + { + MCInst Ldr; + Ldr.setOpcode(AArch64::LDRXui); + Ldr.addOperand(MCOperand::createReg(AArch64::X16)); + Ldr.addOperand(MCOperand::createReg(AArch64::X16)); + MCOperand SymPageOff; + MCInstLowering.lowerOperand( + MachineOperand::CreateES(LazyPointer->getName().data() + 1, + AArch64II::MO_GOT | AArch64II::MO_PAGEOFF), + SymPageOff); + Ldr.addOperand(SymPageOff); + Ldr.addOperand(MCOperand::createImm(0)); + OutStreamer->emitInstruction(Ldr, *STI); + } + + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STRXui) + .addReg(AArch64::X0) + .addReg(AArch64::X16) + .addImm(0), + *STI); + + OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) + .addReg(AArch64::X16) + .addReg(AArch64::X0) + .addImm(0) + .addImm(0), + *STI); + + for (int I = 3; I != -1; --I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost) + .addReg(AArch64::SP) + .addReg(AArch64::D1 + 2 * I) + .addReg(AArch64::D0 + 2 * I) + .addReg(AArch64::SP) + .addImm(2), + *STI); + + for (int I = 3; I != -1; --I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) + .addReg(AArch64::SP) + .addReg(AArch64::X1 + 2 * I) + .addReg(AArch64::X0 + 2 * I) + .addReg(AArch64::SP) + .addImm(2), + *STI); + + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) + .addReg(AArch64::SP) + .addReg(AArch64::FP) + .addReg(AArch64::LR) + .addReg(AArch64::SP) + .addImm(2), + *STI); + + OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() + ? AArch64::BRAAZ + : AArch64::BR) + .addReg(AArch64::X16), + *STI); +} + +void AArch64AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) { + if (!TM.getTargetTriple().isOSBinFormatMachO()) + return AsmPrinter::emitGlobalIFunc(M, GI); + + switch (PreferredIFuncLowering) { + case IFuncLowering::SymbolResolverAlways: + return emitLinkerSymbolResolver(M, GI); + case IFuncLowering::SymbolResolverNever: + return emitManualSymbolResolver(M, GI); + case IFuncLowering::SymbolResolverIfSupported: + if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) + return emitLinkerSymbolResolver(M, GI); + else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) + // NOTE: non-global .symbol_resolvers are not yet supported by Darwin + // linkers + return emitManualSymbolResolver(M, GI); + else + assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); + } +} + // Force static initialization. extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() { RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget()); diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp index 73c7450620966..5241aa6e1c0ea 100644 --- a/llvm/lib/Target/X86/X86AsmPrinter.cpp +++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp @@ -530,6 +530,34 @@ void X86AsmPrinter::PrintIntelMemReference(const MachineInstr *MI, O << ']'; } +void X86AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) { + if (!TM.getTargetTriple().isOSBinFormatMachO()) + return AsmPrinter::emitGlobalIFunc(M, GI); + + OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection()); + + MCSymbol *Name = getSymbol(&GI); + + if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) + OutStreamer->emitSymbolAttribute(Name, MCSA_Global); + else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) + OutStreamer->emitSymbolAttribute(Name, MCSA_WeakReference); + else + assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); + + OutStreamer->emitCodeAlignment(Align(16), Subtarget); + OutStreamer->emitLabel(Name); + OutStreamer->emitSymbolAttribute(Name, MCSA_SymbolResolver); + emitVisibility(Name, GI.getVisibility()); + + // ld64 does not seem to support aliases of symbol resolvers, so we have to + // tail call the resolver manually. + MCInst JMP; + JMP.setOpcode(X86::JMP_4); + JMP.addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))); + OutStreamer->emitInstruction(JMP, *Subtarget); +} + static bool printAsmMRegister(const X86AsmPrinter &P, const MachineOperand &MO, char Mode, raw_ostream &O) { Register Reg = MO.getReg(); diff --git a/llvm/lib/Target/X86/X86AsmPrinter.h b/llvm/lib/Target/X86/X86AsmPrinter.h index c81651cf7f2f0..47f3b7c00c99d 100644 --- a/llvm/lib/Target/X86/X86AsmPrinter.h +++ b/llvm/lib/Target/X86/X86AsmPrinter.h @@ -120,6 +120,7 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter { const char *Modifier); void PrintIntelMemReference(const MachineInstr *MI, unsigned OpNo, raw_ostream &O, const char *Modifier); + void emitGlobalIFunc(Module &M, const GlobalIFunc &GI) override; public: X86AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer); diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-ifunc.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-ifunc.ll new file mode 100644 index 0000000000000..8e51845c2faa9 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-ifunc.ll @@ -0,0 +1,37 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +; RUN: llc -mtriple=aarch64-macho -global-isel -stop-after=irtranslator -verify-machineinstrs -o - %s | FileCheck %s --check-prefixes=MACHO,CHECK +; RUN: llc -mtriple=aarch64-elf -global-isel -stop-after=irtranslator -verify-machineinstrs -o - %s | FileCheck %s --check-prefixes=ELF,CHECK + +@foo_ifunc = ifunc i32 (i32), ptr @foo_resolver + +define internal ptr @foo_resolver() { + ; CHECK-LABEL: name: foo_resolver + ; CHECK: bb.1.entry: + ; CHECK-NEXT: [[C:%[0-9]+]]:_(p0) = G_CONSTANT i64 0 + ; CHECK-NEXT: $x0 = COPY [[C]](p0) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 +entry: + ret ptr null +} + +define void @caller() { + ; MACHO-LABEL: name: caller + ; MACHO: bb.1.entry: + ; MACHO-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp + ; MACHO-NEXT: BL @foo_ifunc, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit-def $w0 + ; MACHO-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp + ; MACHO-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 + ; MACHO-NEXT: RET_ReallyLR + ; + ; ELF-LABEL: name: caller + ; ELF: bb.1.entry: + ; ELF-NEXT: [[GV:%[0-9]+]]:gpr64(p0) = G_GLOBAL_VALUE @foo_ifunc + ; ELF-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp + ; ELF-NEXT: BLR [[GV]](p0), csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit-def $w0 + ; ELF-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp + ; ELF-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 + ; ELF-NEXT: RET_ReallyLR +entry: + %0 = call i32 @foo_ifunc() + ret void +} diff --git a/llvm/test/CodeGen/AArch64/addrsig-macho.ll b/llvm/test/CodeGen/AArch64/addrsig-macho.ll index 360876fccaad3..980b0e7bc4466 100644 --- a/llvm/test/CodeGen/AArch64/addrsig-macho.ll +++ b/llvm/test/CodeGen/AArch64/addrsig-macho.ll @@ -118,8 +118,8 @@ declare void @f3() unnamed_addr @a1 = alias i32, i32* @g1 @a2 = internal local_unnamed_addr alias i32, i32* @g2 -@i1 = ifunc void(), void()* ()* @f1 -@i2 = internal local_unnamed_addr ifunc void(), void()* ()* @f2 +@i1 = external ifunc void(), void()* ()* @f1 +@i2 = external local_unnamed_addr ifunc void(), void()* ()* @f2 declare void @llvm.dbg.value(metadata, metadata, metadata) diff --git a/llvm/test/CodeGen/AArch64/ifunc-asm.ll b/llvm/test/CodeGen/AArch64/ifunc-asm.ll new file mode 100644 index 0000000000000..fbc0f74cee46b --- /dev/null +++ b/llvm/test/CodeGen/AArch64/ifunc-asm.ll @@ -0,0 +1,82 @@ +; RUN: llc -mtriple=arm64-unknown-linux-gnu %s -filetype=asm -o - | FileCheck %s --check-prefixes=ELF +; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=always | FileCheck %s --check-prefixes=MACHO,MACHO-LINKER +; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=if_supported | FileCheck %s --check-prefixes=MACHO,MACHO-DEFAULT +; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=never | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL +; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL + +define internal ptr @foo_resolver() { +entry: + ret ptr null +} +; ELF: .type foo_resolver,@function +; ELF-NEXT: foo_resolver: + +; MACHO: .p2align 2 +; MACHO-NEXT: _foo_resolver + + +@foo_ifunc = ifunc i32 (i32), ptr @foo_resolver +; ELF: .globl foo_ifunc +; ELF-NEXT: .type foo_ifunc,@gnu_indirect_function +; ELF-NEXT: .set foo_ifunc, foo_resolver + +; MACHO-LINKER: .globl _foo_ifunc +; MACHO-LINKER-NEXT: .p2align 2 +; MACHO-LINKER-NEXT: _foo_ifunc: +; MACHO-LINKER-NEXT: .symbol_resolver _foo_ifunc +; MACHO-LINKER-NEXT: b _foo_resolver + +; MACHO-DEFAULT: .globl _foo_ifunc +; MACHO-DEFAULT-NEXT: .p2align 2 +; MACHO-DEFAULT-NEXT: _foo_ifunc: +; MACHO-DEFAULT-NEXT: .symbol_resolver _foo_ifunc +; MACHO-DEFAULT-NEXT: b _foo_resolver + +; MACHO-MANUAL: .section __DATA,__data +; MACHO-MANUAL-NEXT: .globl _foo_ifunc.lazy_pointer +; MACHO-MANUAL-NEXT: _foo_ifunc.lazy_pointer: +; MACHO-MANUAL-NEXT: .quad _foo_ifunc.stub_helper + +; MACHO-MANUAL: .section __TEXT,__text,regular,pure_instructions +; MACHO-MANUAL-NEXT: .globl _foo_ifunc +; MACHO-MANUAL-NEXT: .p2align 2 +; MACHO-MANUAL-NEXT: _foo_ifunc: +; MACHO-MANUAL-NEXT: adrp x16, _foo_ifunc.lazy_pointer@GOTPAGE +; MACHO-MANUAL-NEXT: ldr x16, [x16, _foo_ifunc.lazy_pointer@GOTPAGEOFF] +; MACHO-MANUAL-NEXT: ldr x16, [x16] +; MACHO-MANUAL-NEXT: br x16 +; MACHO-MANUAL-NEXT: .globl _foo_ifunc.stub_helper +; MACHO-MANUAL-NEXT: .p2align 2 +; MACHO-MANUAL-NEXT: _foo_ifunc.stub_helper: +; MACHO-MANUAL-NEXT: stp x29, x30, [sp, #-16]! +; MACHO-MANUAL-NEXT: mov x29, sp +; MACHO-MANUAL-NEXT: stp x1, x0, [sp, #-16]! +; MACHO-MANUAL-NEXT: stp x3, x2, [sp, #-16]! +; MACHO-MANUAL-NEXT: stp x5, x4, [sp, #-16]! +; MACHO-MANUAL-NEXT: stp x7, x6, [sp, #-16]! +; MACHO-MANUAL-NEXT: stp d1, d0, [sp, #-16]! +; MACHO-MANUAL-NEXT: stp d3, d2, [sp, #-16]! +; MACHO-MANUAL-NEXT: stp d5, d4, [sp, #-16]! +; MACHO-MANUAL-NEXT: stp d7, d6, [sp, #-16]! +; MACHO-MANUAL-NEXT: bl _foo_resolver +; MACHO-MANUAL-NEXT: adrp x16, _foo_ifunc.lazy_pointer@GOTPAGE +; MACHO-MANUAL-NEXT: ldr x16, [x16, _foo_ifunc.lazy_pointer@GOTPAGEOFF] +; MACHO-MANUAL-NEXT: str x0, [x16] +; MACHO-MANUAL-NEXT: add x16, x0, #0 +; MACHO-MANUAL-NEXT: ldp d7, d6, [sp], #16 +; MACHO-MANUAL-NEXT: ldp d5, d4, [sp], #16 +; MACHO-MANUAL-NEXT: ldp d3, d2, [sp], #16 +; MACHO-MANUAL-NEXT: ldp d1, d0, [sp], #16 +; MACHO-MANUAL-NEXT: ldp x7, x6, [sp], #16 +; MACHO-MANUAL-NEXT: ldp x5, x4, [sp], #16 +; MACHO-MANUAL-NEXT: ldp x3, x2, [sp], #16 +; MACHO-MANUAL-NEXT: ldp x1, x0, [sp], #16 +; MACHO-MANUAL-NEXT: ldp x29, x30, [sp], #16 +; MACHO-MANUAL-NEXT: br x16 + + +@weak_ifunc = weak ifunc i32 (i32), ptr @foo_resolver +; ELF: .type weak_ifunc,@gnu_indirect_function +; MACHO-LINKER: .symbol_resolver _weak_ifunc +; MACHO-MANUAL: _weak_ifunc.stub_helper: +; MACHO-DEFEAULT: _weak_ifunc.stub_helper: \ No newline at end of file diff --git a/llvm/test/CodeGen/X86/ifunc-asm.ll b/llvm/test/CodeGen/X86/ifunc-asm.ll index 4b380c8ae3301..76efda7115320 100644 --- a/llvm/test/CodeGen/X86/ifunc-asm.ll +++ b/llvm/test/CodeGen/X86/ifunc-asm.ll @@ -1,14 +1,24 @@ -; RUN: llvm-as < %s -o - | llc -filetype=asm | FileCheck %s +; RUN: llc -filetype=asm -mtriple=x86_64-unknown-linux-gnu %s -o - | FileCheck %s --check-prefixes=ELF +; RUN: llc -filetype=asm -mtriple=x86_64-apple-darwin %s -o - | FileCheck %s --check-prefixes=MACHO -target triple = "x86_64-unknown-linux-gnu" - -define internal ptr @foo_ifunc() { +define internal ptr @foo_resolver() { entry: ret ptr null } -; CHECK: .type foo_ifunc,@function -; CHECK-NEXT: foo_ifunc: +; ELF: .type foo_resolver,@function +; ELF-NEXT: foo_resolver: + +; MACHO: .p2align 4, 0x90 +; MACHO-NEXT: _foo_resolver + + +@foo_ifunc = ifunc i32 (i32), ptr @foo_resolver +; ELF: .globl foo_ifunc +; ELF-NEXT: .type foo_ifunc,@gnu_indirect_function +; ELF-NEXT: .set foo_ifunc, foo_resolver -@foo = ifunc i32 (i32), ptr @foo_ifunc -; CHECK: .type foo,@gnu_indirect_function -; CHECK-NEXT: .set foo, foo_ifunc +; MACHO: .globl _foo_ifunc +; MACHO-NEXT: .p2align 4, 0x90 +; MACHO-NEXT: _foo_ifunc: +; MACHO-NEXT: .symbol_resolver _foo_ifunc +; MACHO-NEXT: jmp _foo_resolver diff --git a/llvm/test/Verifier/ifunc-macho.ll b/llvm/test/Verifier/ifunc-macho.ll new file mode 100644 index 0000000000000..2e2166645983a --- /dev/null +++ b/llvm/test/Verifier/ifunc-macho.ll @@ -0,0 +1,42 @@ +; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s + +target triple = "arm64-apple-ios" + +define ptr @resolver() { + ret ptr null +} + +@g = external global i32 +@inval_objtype = ifunc void (), ptr @g +; CHECK: IFunc must have a Function resolver + +declare ptr @resolver_decl() +@inval_resolver_decl = ifunc void (), ptr @resolver_decl +; CHECK: IFunc resolver must be a definition +; CHECK-NEXT: @inval_resolver_decl + +define available_externally ptr @resolver_linker_decl() { + ret ptr null +} +@inval_resolver_decl2 = ifunc void (), ptr @resolver_linker_decl +; CHECK: IFunc resolver must be a definition +; CHECK-NEXT: @inval_resolver_decl2 + +@ifunc_nonpointer_return_type = ifunc i32 (), ptr @resolver_returns_nonpointer +; CHECK: IFunc resolver must return a pointer +; CHECK-NEXT: ptr @ifunc_nonpointer_return_type + +define i32 @resolver_returns_nonpointer() { + ret i32 0 +} + +@valid_external = ifunc void (), ptr @resolver +; CHECK-NOT: valid_external + +@inval_linkonce = linkonce ifunc void (), ptr @resolver + +@inval_weak = weak ifunc void (), ptr @resolver + +@inval_weak_extern = extern_weak ifunc void (), ptr @resolver + +@inval_private = private ifunc void (), ptr @resolver >From 8f6755e6b211c9b0206197f65304443e26e244eb Mon Sep 17 00:00:00 2001 From: Jon Roelofs <jonathan_roel...@apple.com> Date: Tue, 28 Nov 2023 11:04:43 -0800 Subject: [PATCH 2/9] git clang-format Created using spr 1.3.4 --- llvm/include/llvm/CodeGen/AsmPrinter.h | 1 - llvm/lib/IR/Verifier.cpp | 4 +-- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 25 ++++++++++++------- llvm/lib/Target/X86/X86AsmPrinter.cpp | 2 ++ 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h index 48fa6c478464c..07b92871a0f08 100644 --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -887,7 +887,6 @@ class AsmPrinter : public MachineFunctionPass { virtual void emitGlobalIFunc(Module &M, const GlobalIFunc &GI); private: - /// This method decides whether the specified basic block requires a label. bool shouldEmitLabelForBasicBlock(const MachineBasicBlock &MBB) const; diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 94e76a43bf38d..bd90047a411a6 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -959,7 +959,6 @@ void Verifier::visitGlobalIFunc(const GlobalIFunc &GI) { GlobalIFunc::getResolverFunctionType(GI.getValueType()); Check(ResolverTy == ResolverFuncTy->getPointerTo(GI.getAddressSpace()), "IFunc resolver has incorrect type", &GI); - } void Verifier::visitNamedMDNode(const NamedMDNode &NMD) { @@ -2240,7 +2239,8 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, } // Check EVEX512 feature. - if (MaxParameterWidth >= 512 && Attrs.hasFnAttr("target-features") && TT.isX86()) { + if (MaxParameterWidth >= 512 && Attrs.hasFnAttr("target-features") && + TT.isX86()) { StringRef TF = Attrs.getFnAttr("target-features").getValueAsString(); Check(!TF.contains("+avx512f") || !TF.contains("-evex512"), "512-bit vector arguments require 'evex512' for AVX512", V); diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index 2dab8e126c9ab..f4128332008fb 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -71,10 +71,15 @@ using namespace llvm; namespace { -enum class IFuncLowering { SymbolResolverIfSupported, SymbolResolverAlways, SymbolResolverNever }; +enum class IFuncLowering { + SymbolResolverIfSupported, + SymbolResolverAlways, + SymbolResolverNever +}; static cl::opt<IFuncLowering> PreferredIFuncLowering( - "arm64-darwin-ifunc-symbol_resolver", cl::init(IFuncLowering::SymbolResolverNever), + "arm64-darwin-ifunc-symbol_resolver", + cl::init(IFuncLowering::SymbolResolverNever), cl::desc("Pick the lowering for ifuncs on darwin platforms"), cl::Hidden, cl::values( clEnumValN( @@ -1853,8 +1858,8 @@ void AArch64AsmPrinter::emitLinkerSymbolResolver(Module &M, OutStreamer->emitSymbolAttribute(Name, MCSA_SymbolResolver); emitVisibility(Name, GI.getVisibility()); - // ld-prime does not seem to support aliases of symbol resolvers, so we have to - // tail call the resolver manually. + // ld-prime does not seem to support aliases of symbol resolvers, so we have + // to tail call the resolver manually. OutStreamer->emitInstruction( MCInstBuilder(AArch64::B) .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), @@ -1887,8 +1892,9 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); }; - MCSymbol *LazyPointer = TM.getObjFileLowering()->getContext().getOrCreateSymbol( - "_" + GI.getName() + ".lazy_pointer"); + MCSymbol *LazyPointer = + TM.getObjFileLowering()->getContext().getOrCreateSymbol( + "_" + GI.getName() + ".lazy_pointer"); MCSymbol *StubHelper = TM.getObjFileLowering()->getContext().getOrCreateSymbol( "_" + GI.getName() + ".stub_helper"); @@ -1943,9 +1949,10 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, } OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRXui) - .addReg(AArch64::X16) - .addReg(AArch64::X16) - .addImm(0), *STI); + .addReg(AArch64::X16) + .addReg(AArch64::X16) + .addImm(0), + *STI); OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp index 5241aa6e1c0ea..37158900d2404 100644 --- a/llvm/lib/Target/X86/X86AsmPrinter.cpp +++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp @@ -556,6 +556,8 @@ void X86AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) { JMP.setOpcode(X86::JMP_4); JMP.addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))); OutStreamer->emitInstruction(JMP, *Subtarget); + + // FIXME: do the manual .symbol_resolver lowering that we did in AArch64AsmPrinter. } static bool printAsmMRegister(const X86AsmPrinter &P, const MachineOperand &MO, >From dc1fcb464e3d9e8dd37a5bda1aa8894e127654e9 Mon Sep 17 00:00:00 2001 From: Jon Roelofs <jonathan_roel...@apple.com> Date: Tue, 28 Nov 2023 14:24:45 -0800 Subject: [PATCH 3/9] avoid writeback stp's Created using spr 1.3.4 --- llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 48 ++++---- llvm/lib/CodeGen/GlobalISel/CallLowering.cpp | 2 +- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 110 ++++++++++-------- llvm/test/CodeGen/AArch64/addrsig-macho.ll | 17 ++- llvm/test/CodeGen/AArch64/ifunc-asm.ll | 99 ++++++++-------- llvm/test/Verifier/ifunc-macho.ll | 42 ------- 6 files changed, 153 insertions(+), 165 deletions(-) delete mode 100644 llvm/test/Verifier/ifunc-macho.ll diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 15ff398836803..b4ac0a70e7fde 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -2300,6 +2300,32 @@ bool AsmPrinter::doFinalization(Module &M) { // through user plugins. emitStackMaps(); + // Print aliases in topological order, that is, for each alias a = b, + // b must be printed before a. + // This is because on some targets (e.g. PowerPC) linker expects aliases in + // such an order to generate correct TOC information. + SmallVector<const GlobalAlias *, 16> AliasStack; + SmallPtrSet<const GlobalAlias *, 16> AliasVisited; + for (const auto &Alias : M.aliases()) { + if (Alias.hasAvailableExternallyLinkage()) + continue; + for (const GlobalAlias *Cur = &Alias; Cur; + Cur = dyn_cast<GlobalAlias>(Cur->getAliasee())) { + if (!AliasVisited.insert(Cur).second) + break; + AliasStack.push_back(Cur); + } + for (const GlobalAlias *AncestorAlias : llvm::reverse(AliasStack)) + emitGlobalAlias(M, *AncestorAlias); + AliasStack.clear(); + } + + // IFuncs must come before deubginfo in case the backend decides to emit them + // as actual functions, since on MachO targets, we cannot create regular + // sections after DWARF. + for (const auto &IFunc : M.ifuncs()) + emitGlobalIFunc(M, IFunc); + // Finalize debug and EH information. for (const HandlerInfo &HI : Handlers) { NamedRegionTimer T(HI.TimerName, HI.TimerDescription, HI.TimerGroupName, @@ -2339,28 +2365,6 @@ bool AsmPrinter::doFinalization(Module &M) { } } - // Print aliases in topological order, that is, for each alias a = b, - // b must be printed before a. - // This is because on some targets (e.g. PowerPC) linker expects aliases in - // such an order to generate correct TOC information. - SmallVector<const GlobalAlias *, 16> AliasStack; - SmallPtrSet<const GlobalAlias *, 16> AliasVisited; - for (const auto &Alias : M.aliases()) { - if (Alias.hasAvailableExternallyLinkage()) - continue; - for (const GlobalAlias *Cur = &Alias; Cur; - Cur = dyn_cast<GlobalAlias>(Cur->getAliasee())) { - if (!AliasVisited.insert(Cur).second) - break; - AliasStack.push_back(Cur); - } - for (const GlobalAlias *AncestorAlias : llvm::reverse(AliasStack)) - emitGlobalAlias(M, *AncestorAlias); - AliasStack.clear(); - } - for (const auto &IFunc : M.ifuncs()) - emitGlobalIFunc(M, IFunc); - GCModuleInfo *MI = getAnalysisIfAvailable<GCModuleInfo>(); assert(MI && "AsmPrinter didn't require GCModuleInfo?"); for (GCModuleInfo::iterator I = MI->end(), E = MI->begin(); I != E; ) diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp index e0080b145d4f9..ce736178afc8b 100644 --- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -147,7 +147,7 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, if (const GlobalIFunc *IF = dyn_cast<GlobalIFunc>(CalleeV); IF && MF.getTarget().getTargetTriple().isOSBinFormatMachO()) { // ld64 requires that .symbol_resolvers to be called via a stub, so these - // must always be a diret call. + // must always be a direct call. Info.Callee = MachineOperand::CreateGA(IF, 0); } else if (const Function *F = dyn_cast<Function>(CalleeV)) Info.Callee = MachineOperand::CreateGA(F, 0); diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index f4128332008fb..26b3a14e22b2a 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -1965,65 +1965,71 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, OutStreamer->emitLabel(StubHelper); emitVisibility(StubHelper, GI.getVisibility()); - // stp fp, lr, [sp, #-16]! - // mov fp, sp - // stp x1, x0, [sp, #-16]! - // stp x3, x2, [sp, #-16]! - // stp x5, x4, [sp, #-16]! - // stp x7, x6, [sp, #-16]! - // stp d1, d0, [sp, #-16]! - // stp d3, d2, [sp, #-16]! - // stp d5, d4, [sp, #-16]! - // stp d7, d6, [sp, #-16]! + // stp fp, lr, [sp, #-16] + // sub fp, sp, 16 + // stp x1, x0, [sp, #-32] + // stp x3, x2, [sp, #-48] + // stp x5, x4, [sp, #-64] + // stp x7, x6, [sp, #-80] + // stp d1, d0, [sp, #-96] + // stp d3, d2, [sp, #-112] + // stp d5, d4, [sp, #-128] + // stp d7, d6, [sp, #-144] + // sub sp, sp, 144 // bl _resolver // adrp x16, lazy_pointer@GOTPAGE // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] // str x0, [x16] // mov x16, x0 - // ldp d7, d6, [sp], #16 - // ldp d5, d4, [sp], #16 - // ldp d3, d2, [sp], #16 - // ldp d1, d0, [sp], #16 - // ldp x7, x6, [sp], #16 - // ldp x5, x4, [sp], #16 - // ldp x3, x2, [sp], #16 - // ldp x1, x0, [sp], #16 - // ldp fp, lr, [sp], #16 + // add sp, sp, 144 + // ldp d7, d6, [sp, #-144] + // ldp d5, d4, [sp, #-128] + // ldp d3, d2, [sp, #-112] + // ldp d1, d0, [sp, #-96] + // ldp x7, x6, [sp, #-80] + // ldp x5, x4, [sp, #-64] + // ldp x3, x2, [sp, #-48] + // ldp x1, x0, [sp, #-32] + // ldp fp, lr, [sp, #-16] // br x16 - OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) - .addReg(AArch64::SP) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi) .addReg(AArch64::FP) .addReg(AArch64::LR) .addReg(AArch64::SP) .addImm(-2), *STI); - OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBXri) .addReg(AArch64::FP) .addReg(AArch64::SP) - .addImm(0) + .addImm(16) .addImm(0), *STI); - for (int I = 0; I != 4; ++I) - OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) - .addReg(AArch64::SP) - .addReg(AArch64::X1 + 2 * I) - .addReg(AArch64::X0 + 2 * I) + for (int I = 0; I != 8; I += 2) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi) + .addReg(AArch64::X1 + I) + .addReg(AArch64::X0 + I) .addReg(AArch64::SP) - .addImm(-2), + .addImm(-4 - I), *STI); - for (int I = 0; I != 4; ++I) - OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre) + for (int I = 0; I != 8; I += 2) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDi) + .addReg(AArch64::D1 + I) + .addReg(AArch64::D0 + I) .addReg(AArch64::SP) - .addReg(AArch64::D1 + 2 * I) - .addReg(AArch64::D0 + 2 * I) - .addReg(AArch64::SP) - .addImm(-2), + .addImm(-12 - I), *STI); + OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBXri) + .addReg(AArch64::SP) + .addReg(AArch64::SP) + .addImm(144) + .addImm(0), + *STI); + OutStreamer->emitInstruction( MCInstBuilder(AArch64::BL) .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), @@ -2070,30 +2076,34 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, .addImm(0), *STI); - for (int I = 3; I != -1; --I) - OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost) - .addReg(AArch64::SP) - .addReg(AArch64::D1 + 2 * I) - .addReg(AArch64::D0 + 2 * I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) + .addReg(AArch64::SP) + .addReg(AArch64::SP) + .addImm(144) + .addImm(0), + *STI); + + for (int I = 6; I != -2; I -= 2) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDi) + .addReg(AArch64::D1 + I) + .addReg(AArch64::D0 + I) .addReg(AArch64::SP) - .addImm(2), + .addImm(-12 - I), *STI); - for (int I = 3; I != -1; --I) - OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) - .addReg(AArch64::SP) - .addReg(AArch64::X1 + 2 * I) - .addReg(AArch64::X0 + 2 * I) + for (int I = 6; I != -2; I -= 2) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXi) + .addReg(AArch64::X1 + I) + .addReg(AArch64::X0 + I) .addReg(AArch64::SP) - .addImm(2), + .addImm(-4 - I), *STI); - OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) - .addReg(AArch64::SP) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXi) .addReg(AArch64::FP) .addReg(AArch64::LR) .addReg(AArch64::SP) - .addImm(2), + .addImm(-2), *STI); OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() diff --git a/llvm/test/CodeGen/AArch64/addrsig-macho.ll b/llvm/test/CodeGen/AArch64/addrsig-macho.ll index 980b0e7bc4466..62bc764e0251b 100644 --- a/llvm/test/CodeGen/AArch64/addrsig-macho.ll +++ b/llvm/test/CodeGen/AArch64/addrsig-macho.ll @@ -3,6 +3,19 @@ ; RUN: llvm-objdump --macho --section-headers %t | FileCheck %s --check-prefix=SECTIONS ; RUN: llvm-objdump --macho --reloc %t | FileCheck %s --check-prefix=RELOCS +; CHECK: .section __DATA,__data +; CHECK: _i1.lazy_pointer: +; CHECK: .section __TEXT,__text,regular,pure_instructions +; CHECK: _i1: +; CHECK: _i1.stub_helper: +; CHECK: .section __DATA,__data +; CHECK: _i2.lazy_pointer: +; CHECK: .section __TEXT,__text,regular,pure_instructions +; CHECK: _i2: +; CHECK: _i2.stub_helper: + +; CHECK: .section __DWARF + ; CHECK: .addrsig{{$}} ; CHECK-NEXT: .addrsig_sym _func03_takeaddr ; CHECK-NEXT: .addrsig_sym _f1 @@ -118,8 +131,8 @@ declare void @f3() unnamed_addr @a1 = alias i32, i32* @g1 @a2 = internal local_unnamed_addr alias i32, i32* @g2 -@i1 = external ifunc void(), void()* ()* @f1 -@i2 = external local_unnamed_addr ifunc void(), void()* ()* @f2 +@i1 = ifunc void(), void()* ()* @f1 +@i2 = internal local_unnamed_addr ifunc void(), void()* ()* @f2 declare void @llvm.dbg.value(metadata, metadata, metadata) diff --git a/llvm/test/CodeGen/AArch64/ifunc-asm.ll b/llvm/test/CodeGen/AArch64/ifunc-asm.ll index fbc0f74cee46b..ede669aa52703 100644 --- a/llvm/test/CodeGen/AArch64/ifunc-asm.ll +++ b/llvm/test/CodeGen/AArch64/ifunc-asm.ll @@ -3,79 +3,82 @@ ; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=if_supported | FileCheck %s --check-prefixes=MACHO,MACHO-DEFAULT ; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=never | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL ; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL +; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -global-isel | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL -define internal ptr @foo_resolver() { +define internal ptr @the_resolver() { entry: ret ptr null } -; ELF: .type foo_resolver,@function -; ELF-NEXT: foo_resolver: +; ELF: .type the_resolver,@function +; ELF-NEXT: the_resolver: ; MACHO: .p2align 2 -; MACHO-NEXT: _foo_resolver +; MACHO-NEXT: _the_resolver -@foo_ifunc = ifunc i32 (i32), ptr @foo_resolver -; ELF: .globl foo_ifunc -; ELF-NEXT: .type foo_ifunc,@gnu_indirect_function -; ELF-NEXT: .set foo_ifunc, foo_resolver +@global_ifunc = ifunc i32 (i32), ptr @the_resolver +; ELF: .globl global_ifunc +; ELF-NEXT: .type global_ifunc,@gnu_indirect_function +; ELF-NEXT: .set global_ifunc, the_resolver -; MACHO-LINKER: .globl _foo_ifunc +; MACHO-LINKER: .globl _global_ifunc ; MACHO-LINKER-NEXT: .p2align 2 -; MACHO-LINKER-NEXT: _foo_ifunc: -; MACHO-LINKER-NEXT: .symbol_resolver _foo_ifunc -; MACHO-LINKER-NEXT: b _foo_resolver +; MACHO-LINKER-NEXT: _global_ifunc: +; MACHO-LINKER-NEXT: .symbol_resolver _global_ifunc +; MACHO-LINKER-NEXT: b _the_resolver -; MACHO-DEFAULT: .globl _foo_ifunc +; MACHO-DEFAULT: .globl _global_ifunc ; MACHO-DEFAULT-NEXT: .p2align 2 -; MACHO-DEFAULT-NEXT: _foo_ifunc: -; MACHO-DEFAULT-NEXT: .symbol_resolver _foo_ifunc -; MACHO-DEFAULT-NEXT: b _foo_resolver +; MACHO-DEFAULT-NEXT: _global_ifunc: +; MACHO-DEFAULT-NEXT: .symbol_resolver _global_ifunc +; MACHO-DEFAULT-NEXT: b _the_resolver ; MACHO-MANUAL: .section __DATA,__data -; MACHO-MANUAL-NEXT: .globl _foo_ifunc.lazy_pointer -; MACHO-MANUAL-NEXT: _foo_ifunc.lazy_pointer: -; MACHO-MANUAL-NEXT: .quad _foo_ifunc.stub_helper +; MACHO-MANUAL-NEXT: .globl _global_ifunc.lazy_pointer +; MACHO-MANUAL-NEXT: _global_ifunc.lazy_pointer: +; MACHO-MANUAL-NEXT: .quad _global_ifunc.stub_helper ; MACHO-MANUAL: .section __TEXT,__text,regular,pure_instructions -; MACHO-MANUAL-NEXT: .globl _foo_ifunc +; MACHO-MANUAL-NEXT: .globl _global_ifunc ; MACHO-MANUAL-NEXT: .p2align 2 -; MACHO-MANUAL-NEXT: _foo_ifunc: -; MACHO-MANUAL-NEXT: adrp x16, _foo_ifunc.lazy_pointer@GOTPAGE -; MACHO-MANUAL-NEXT: ldr x16, [x16, _foo_ifunc.lazy_pointer@GOTPAGEOFF] +; MACHO-MANUAL-NEXT: _global_ifunc: +; MACHO-MANUAL-NEXT: adrp x16, _global_ifunc.lazy_pointer@GOTPAGE +; MACHO-MANUAL-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer@GOTPAGEOFF] ; MACHO-MANUAL-NEXT: ldr x16, [x16] ; MACHO-MANUAL-NEXT: br x16 -; MACHO-MANUAL-NEXT: .globl _foo_ifunc.stub_helper +; MACHO-MANUAL-NEXT: .globl _global_ifunc.stub_helper ; MACHO-MANUAL-NEXT: .p2align 2 -; MACHO-MANUAL-NEXT: _foo_ifunc.stub_helper: -; MACHO-MANUAL-NEXT: stp x29, x30, [sp, #-16]! -; MACHO-MANUAL-NEXT: mov x29, sp -; MACHO-MANUAL-NEXT: stp x1, x0, [sp, #-16]! -; MACHO-MANUAL-NEXT: stp x3, x2, [sp, #-16]! -; MACHO-MANUAL-NEXT: stp x5, x4, [sp, #-16]! -; MACHO-MANUAL-NEXT: stp x7, x6, [sp, #-16]! -; MACHO-MANUAL-NEXT: stp d1, d0, [sp, #-16]! -; MACHO-MANUAL-NEXT: stp d3, d2, [sp, #-16]! -; MACHO-MANUAL-NEXT: stp d5, d4, [sp, #-16]! -; MACHO-MANUAL-NEXT: stp d7, d6, [sp, #-16]! -; MACHO-MANUAL-NEXT: bl _foo_resolver -; MACHO-MANUAL-NEXT: adrp x16, _foo_ifunc.lazy_pointer@GOTPAGE -; MACHO-MANUAL-NEXT: ldr x16, [x16, _foo_ifunc.lazy_pointer@GOTPAGEOFF] +; MACHO-MANUAL-NEXT: _global_ifunc.stub_helper: +; MACHO-MANUAL-NEXT: stp x29, x30, [sp, #-16] +; MACHO-MANUAL-NEXT: sub x29, sp, #16 +; MACHO-MANUAL-NEXT: stp x1, x0, [sp, #-32] +; MACHO-MANUAL-NEXT: stp x3, x2, [sp, #-48] +; MACHO-MANUAL-NEXT: stp x5, x4, [sp, #-64] +; MACHO-MANUAL-NEXT: stp x7, x6, [sp, #-80] +; MACHO-MANUAL-NEXT: stp d1, d0, [sp, #-96] +; MACHO-MANUAL-NEXT: stp d3, d2, [sp, #-112] +; MACHO-MANUAL-NEXT: stp d5, d4, [sp, #-128] +; MACHO-MANUAL-NEXT: stp d7, d6, [sp, #-144] +; MACHO-MANUAL-NEXT: sub sp, sp, #144 +; MACHO-MANUAL-NEXT: bl _the_resolver +; MACHO-MANUAL-NEXT: adrp x16, _global_ifunc.lazy_pointer@GOTPAGE +; MACHO-MANUAL-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer@GOTPAGEOFF] ; MACHO-MANUAL-NEXT: str x0, [x16] ; MACHO-MANUAL-NEXT: add x16, x0, #0 -; MACHO-MANUAL-NEXT: ldp d7, d6, [sp], #16 -; MACHO-MANUAL-NEXT: ldp d5, d4, [sp], #16 -; MACHO-MANUAL-NEXT: ldp d3, d2, [sp], #16 -; MACHO-MANUAL-NEXT: ldp d1, d0, [sp], #16 -; MACHO-MANUAL-NEXT: ldp x7, x6, [sp], #16 -; MACHO-MANUAL-NEXT: ldp x5, x4, [sp], #16 -; MACHO-MANUAL-NEXT: ldp x3, x2, [sp], #16 -; MACHO-MANUAL-NEXT: ldp x1, x0, [sp], #16 -; MACHO-MANUAL-NEXT: ldp x29, x30, [sp], #16 +; MACHO-MANUAL-NEXT: add sp, sp, #144 +; MACHO-MANUAL-NEXT: ldp d7, d6, [sp, #-144] +; MACHO-MANUAL-NEXT: ldp d5, d4, [sp, #-128] +; MACHO-MANUAL-NEXT: ldp d3, d2, [sp, #-112] +; MACHO-MANUAL-NEXT: ldp d1, d0, [sp, #-96] +; MACHO-MANUAL-NEXT: ldp x7, x6, [sp, #-80] +; MACHO-MANUAL-NEXT: ldp x5, x4, [sp, #-64] +; MACHO-MANUAL-NEXT: ldp x3, x2, [sp, #-48] +; MACHO-MANUAL-NEXT: ldp x1, x0, [sp, #-32] +; MACHO-MANUAL-NEXT: ldp x29, x30, [sp, #-16] ; MACHO-MANUAL-NEXT: br x16 -@weak_ifunc = weak ifunc i32 (i32), ptr @foo_resolver +@weak_ifunc = weak ifunc i32 (i32), ptr @the_resolver ; ELF: .type weak_ifunc,@gnu_indirect_function ; MACHO-LINKER: .symbol_resolver _weak_ifunc ; MACHO-MANUAL: _weak_ifunc.stub_helper: diff --git a/llvm/test/Verifier/ifunc-macho.ll b/llvm/test/Verifier/ifunc-macho.ll deleted file mode 100644 index 2e2166645983a..0000000000000 --- a/llvm/test/Verifier/ifunc-macho.ll +++ /dev/null @@ -1,42 +0,0 @@ -; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s - -target triple = "arm64-apple-ios" - -define ptr @resolver() { - ret ptr null -} - -@g = external global i32 -@inval_objtype = ifunc void (), ptr @g -; CHECK: IFunc must have a Function resolver - -declare ptr @resolver_decl() -@inval_resolver_decl = ifunc void (), ptr @resolver_decl -; CHECK: IFunc resolver must be a definition -; CHECK-NEXT: @inval_resolver_decl - -define available_externally ptr @resolver_linker_decl() { - ret ptr null -} -@inval_resolver_decl2 = ifunc void (), ptr @resolver_linker_decl -; CHECK: IFunc resolver must be a definition -; CHECK-NEXT: @inval_resolver_decl2 - -@ifunc_nonpointer_return_type = ifunc i32 (), ptr @resolver_returns_nonpointer -; CHECK: IFunc resolver must return a pointer -; CHECK-NEXT: ptr @ifunc_nonpointer_return_type - -define i32 @resolver_returns_nonpointer() { - ret i32 0 -} - -@valid_external = ifunc void (), ptr @resolver -; CHECK-NOT: valid_external - -@inval_linkonce = linkonce ifunc void (), ptr @resolver - -@inval_weak = weak ifunc void (), ptr @resolver - -@inval_weak_extern = extern_weak ifunc void (), ptr @resolver - -@inval_private = private ifunc void (), ptr @resolver >From bbeb3beaf37d91911a96a20bb1825fa16dcfc094 Mon Sep 17 00:00:00 2001 From: Jon Roelofs <jonathan_roel...@apple.com> Date: Wed, 29 Nov 2023 08:23:21 -0800 Subject: [PATCH 4/9] x86 support Created using spr 1.3.4 --- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 72 +++++----- llvm/lib/Target/X86/X86AsmPrinter.cpp | 124 +++++++++++++++--- llvm/test/CodeGen/X86/ifunc-asm.ll | 48 +++++-- 3 files changed, 178 insertions(+), 66 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index 26b3a14e22b2a..1b5b7c556c79f 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -1901,6 +1901,9 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, OutStreamer->switchSection(OutContext.getObjectFileInfo()->getDataSection()); + // _ifunc.lazy_pointer: + // .quad _ifunc.stub_helper + EmitLinkage(LazyPointer); OutStreamer->emitLabel(LazyPointer); emitVisibility(LazyPointer, GI.getVisibility()); @@ -1908,18 +1911,18 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection()); - MCSymbol *Stub = getSymbol(&GI); + // _ifunc: + // adrp x16, lazy_pointer@GOTPAGE + // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] + // ldr x16, [x16] + // br x16 + MCSymbol *Stub = getSymbol(&GI); EmitLinkage(Stub); OutStreamer->emitCodeAlignment(Align(4), STI); OutStreamer->emitLabel(Stub); emitVisibility(Stub, GI.getVisibility()); - // adrp x16, lazy_pointer@GOTPAGE - // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] - // ldr x16, [x16] - // br x16 - { MCInst Adrp; Adrp.setOpcode(AArch64::ADRP); @@ -1960,39 +1963,40 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, .addReg(AArch64::X16), *STI); + // _ifunc.stub_helper: + // stp fp, lr, [sp, #-16] + // sub fp, sp, 16 + // stp x1, x0, [sp, #-32] + // stp x3, x2, [sp, #-48] + // stp x5, x4, [sp, #-64] + // stp x7, x6, [sp, #-80] + // stp d1, d0, [sp, #-96] + // stp d3, d2, [sp, #-112] + // stp d5, d4, [sp, #-128] + // stp d7, d6, [sp, #-144] + // sub sp, sp, 144 + // bl _resolver + // adrp x16, lazy_pointer@GOTPAGE + // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] + // str x0, [x16] + // mov x16, x0 + // add sp, sp, 144 + // ldp d7, d6, [sp, #-144] + // ldp d5, d4, [sp, #-128] + // ldp d3, d2, [sp, #-112] + // ldp d1, d0, [sp, #-96] + // ldp x7, x6, [sp, #-80] + // ldp x5, x4, [sp, #-64] + // ldp x3, x2, [sp, #-48] + // ldp x1, x0, [sp, #-32] + // ldp fp, lr, [sp, #-16] + // br x16 + EmitLinkage(StubHelper); OutStreamer->emitCodeAlignment(Align(4), STI); OutStreamer->emitLabel(StubHelper); emitVisibility(StubHelper, GI.getVisibility()); - // stp fp, lr, [sp, #-16] - // sub fp, sp, 16 - // stp x1, x0, [sp, #-32] - // stp x3, x2, [sp, #-48] - // stp x5, x4, [sp, #-64] - // stp x7, x6, [sp, #-80] - // stp d1, d0, [sp, #-96] - // stp d3, d2, [sp, #-112] - // stp d5, d4, [sp, #-128] - // stp d7, d6, [sp, #-144] - // sub sp, sp, 144 - // bl _resolver - // adrp x16, lazy_pointer@GOTPAGE - // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] - // str x0, [x16] - // mov x16, x0 - // add sp, sp, 144 - // ldp d7, d6, [sp, #-144] - // ldp d5, d4, [sp, #-128] - // ldp d3, d2, [sp, #-112] - // ldp d1, d0, [sp, #-96] - // ldp x7, x6, [sp, #-80] - // ldp x5, x4, [sp, #-64] - // ldp x3, x2, [sp, #-48] - // ldp x1, x0, [sp, #-32] - // ldp fp, lr, [sp, #-16] - // br x16 - OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi) .addReg(AArch64::FP) .addReg(AArch64::LR) diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp index 37158900d2404..b0f4b9d984372 100644 --- a/llvm/lib/Target/X86/X86AsmPrinter.cpp +++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp @@ -14,6 +14,7 @@ #include "X86AsmPrinter.h" #include "MCTargetDesc/X86ATTInstPrinter.h" #include "MCTargetDesc/X86BaseInfo.h" +#include "MCTargetDesc/X86MCTargetDesc.h" #include "MCTargetDesc/X86TargetStreamer.h" #include "TargetInfo/X86TargetInfo.h" #include "X86InstrInfo.h" @@ -34,6 +35,7 @@ #include "llvm/MC/MCCodeEmitter.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstBuilder.h" #include "llvm/MC/MCSectionCOFF.h" #include "llvm/MC/MCSectionELF.h" @@ -534,30 +536,112 @@ void X86AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) { if (!TM.getTargetTriple().isOSBinFormatMachO()) return AsmPrinter::emitGlobalIFunc(M, GI); - OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection()); + auto EmitLinkage = [&](MCSymbol *Sym) { + if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) + OutStreamer->emitSymbolAttribute(Sym, MCSA_Global); + else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) + OutStreamer->emitSymbolAttribute(Sym, MCSA_WeakReference); + else + assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); + }; - MCSymbol *Name = getSymbol(&GI); + MCSymbol *LazyPointer = + TM.getObjFileLowering()->getContext().getOrCreateSymbol( + "_" + GI.getName() + ".lazy_pointer"); + MCSymbol *StubHelper = + TM.getObjFileLowering()->getContext().getOrCreateSymbol( + "_" + GI.getName() + ".stub_helper"); - if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) - OutStreamer->emitSymbolAttribute(Name, MCSA_Global); - else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) - OutStreamer->emitSymbolAttribute(Name, MCSA_WeakReference); - else - assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); + OutStreamer->switchSection(OutContext.getObjectFileInfo()->getDataSection()); + + // _ifunc.lazy_pointer: + // .quad _ifunc.stub_helper + EmitLinkage(LazyPointer); + OutStreamer->emitLabel(LazyPointer); + emitVisibility(LazyPointer, GI.getVisibility()); + OutStreamer->emitValue(MCSymbolRefExpr::create(StubHelper, OutContext), 8); + + OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection()); + + // _ifunc: + // jmpq *lazy_pointer(%rip) + + MCSymbol *Stub = getSymbol(&GI); + EmitLinkage(Stub); + OutStreamer->emitCodeAlignment(Align(16), Subtarget); + OutStreamer->emitLabel(Stub); + emitVisibility(Stub, GI.getVisibility()); + + OutStreamer->emitInstruction( + MCInstBuilder(X86::JMP32m) + .addReg(X86::RIP) + .addImm(1) + .addReg(0) + .addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(LazyPointer, OutContext))) + .addReg(0), + *Subtarget); + + // _ifunc.stub_helper: + // push %rax + // push %rdi + // push %rsi + // push %rdx + // push %rcx + // push %r8 + // push %r9 + // callq foo + // movq %rax,lazy_pointer(%rip) + // pop %r9 + // pop %r8 + // pop %rcx + // pop %rdx + // pop %rsi + // pop %rdi + // pop %rax + // jmpq *lazy_pointer(%rip) + + EmitLinkage(StubHelper); OutStreamer->emitCodeAlignment(Align(16), Subtarget); - OutStreamer->emitLabel(Name); - OutStreamer->emitSymbolAttribute(Name, MCSA_SymbolResolver); - emitVisibility(Name, GI.getVisibility()); - - // ld64 does not seem to support aliases of symbol resolvers, so we have to - // tail call the resolver manually. - MCInst JMP; - JMP.setOpcode(X86::JMP_4); - JMP.addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))); - OutStreamer->emitInstruction(JMP, *Subtarget); - - // FIXME: do the manual .symbol_resolver lowering that we did in AArch64AsmPrinter. + OutStreamer->emitLabel(StubHelper); + emitVisibility(StubHelper, GI.getVisibility()); + + for (int Reg : + {X86::RAX, X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9}) + OutStreamer->emitInstruction(MCInstBuilder(X86::PUSH64r).addReg(Reg), + *Subtarget); + + OutStreamer->emitInstruction( + MCInstBuilder(X86::CALL64pcrel32) + .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), + *Subtarget); + + OutStreamer->emitInstruction( + MCInstBuilder(X86::MOV64mr) + .addReg(X86::RIP) + .addImm(1) + .addReg(0) + .addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(LazyPointer, OutContext))) + .addReg(0) + .addReg(X86::RAX), + *Subtarget); + + for (int Reg : + {X86::R9, X86::R8, X86::RCX, X86::RDX, X86::RSI, X86::RDI, X86::RAX}) + OutStreamer->emitInstruction(MCInstBuilder(X86::POP64r).addReg(Reg), + *Subtarget); + + OutStreamer->emitInstruction( + MCInstBuilder(X86::JMP32m) + .addReg(X86::RIP) + .addImm(1) + .addReg(0) + .addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(LazyPointer, OutContext))) + .addReg(0), + *Subtarget); } static bool printAsmMRegister(const X86AsmPrinter &P, const MachineOperand &MO, diff --git a/llvm/test/CodeGen/X86/ifunc-asm.ll b/llvm/test/CodeGen/X86/ifunc-asm.ll index 76efda7115320..0f66febbe95b2 100644 --- a/llvm/test/CodeGen/X86/ifunc-asm.ll +++ b/llvm/test/CodeGen/X86/ifunc-asm.ll @@ -5,20 +5,44 @@ define internal ptr @foo_resolver() { entry: ret ptr null } -; ELF: .type foo_resolver,@function -; ELF-NEXT: foo_resolver: +; ELF: .type foo_resolver,@function +; ELF-NEXT: foo_resolver: -; MACHO: .p2align 4, 0x90 -; MACHO-NEXT: _foo_resolver +; MACHO: .p2align 4, 0x90 +; MACHO-NEXT: _foo_resolver @foo_ifunc = ifunc i32 (i32), ptr @foo_resolver -; ELF: .globl foo_ifunc -; ELF-NEXT: .type foo_ifunc,@gnu_indirect_function -; ELF-NEXT: .set foo_ifunc, foo_resolver +; ELF: .globl foo_ifunc +; ELF-NEXT: .type foo_ifunc,@gnu_indirect_function +; ELF-NEXT: .set foo_ifunc, foo_resolver -; MACHO: .globl _foo_ifunc -; MACHO-NEXT: .p2align 4, 0x90 -; MACHO-NEXT: _foo_ifunc: -; MACHO-NEXT: .symbol_resolver _foo_ifunc -; MACHO-NEXT: jmp _foo_resolver +; MACHO: .section __DATA,__data +; MACHO-NEXT: .globl _foo_ifunc.lazy_pointer +; MACHO-NEXT: _foo_ifunc.lazy_pointer: +; MACHO-NEXT: .quad _foo_ifunc.stub_helper +; MACHO-NEXT: .section __TEXT,__text,regular,pure_instructions +; MACHO-NEXT: .globl _foo_ifunc +; MACHO-NEXT: .p2align 4, 0x90 +; MACHO-NEXT: _foo_ifunc: +; MACHO-NEXT: jmpl *_foo_ifunc.lazy_pointer(%rip) +; MACHO-NEXT: .globl _foo_ifunc.stub_helper +; MACHO-NEXT: .p2align 4, 0x90 +; MACHO-NEXT: _foo_ifunc.stub_helper: +; MACHO-NEXT: pushq %rax +; MACHO-NEXT: pushq %rdi +; MACHO-NEXT: pushq %rsi +; MACHO-NEXT: pushq %rdx +; MACHO-NEXT: pushq %rcx +; MACHO-NEXT: pushq %r8 +; MACHO-NEXT: pushq %r9 +; MACHO-NEXT: callq _foo_resolver +; MACHO-NEXT: movq %rax, _foo_ifunc.lazy_pointer(%rip) +; MACHO-NEXT: popq %r9 +; MACHO-NEXT: popq %r8 +; MACHO-NEXT: popq %rcx +; MACHO-NEXT: popq %rdx +; MACHO-NEXT: popq %rsi +; MACHO-NEXT: popq %rdi +; MACHO-NEXT: popq %rax +; MACHO-NEXT: jmpl *_foo_ifunc.lazy_pointer(%rip) \ No newline at end of file >From b2d46d62984b685cd94597c957708e7b5b34d671 Mon Sep 17 00:00:00 2001 From: Jon Roelofs <jonathan_roel...@apple.com> Date: Wed, 29 Nov 2023 13:56:55 -0800 Subject: [PATCH 5/9] review feedback Created using spr 1.3.4 --- llvm/docs/LangRef.rst | 2 +- llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index cb222e979db29..fece4ac7f127d 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -937,7 +937,7 @@ IFuncs, like as aliases, don't create any new data or func. They are just a new symbol that is resolved at runtime by calling a resolver function. On ELF platforms, IFuncs are resolved by the dynamic linker at load time. On -MachO platforms, they are lowered in terms of ``.symbol_resolver``s, which +Mach-O platforms, they are lowered in terms of ``.symbol_resolver``s, which lazily resolve the callee the first time they are called. IFunc may have an optional :ref:`linkage type <linkage>` and an optional diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index b4ac0a70e7fde..aaa7693c61f0e 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -2321,7 +2321,7 @@ bool AsmPrinter::doFinalization(Module &M) { } // IFuncs must come before deubginfo in case the backend decides to emit them - // as actual functions, since on MachO targets, we cannot create regular + // as actual functions, since on Mach-O targets, we cannot create regular // sections after DWARF. for (const auto &IFunc : M.ifuncs()) emitGlobalIFunc(M, IFunc); >From e6af9eaa469fa0f2cf3b1df7ec80bed3c2354cbe Mon Sep 17 00:00:00 2001 From: Jon Roelofs <jonathan_roel...@apple.com> Date: Wed, 29 Nov 2023 14:07:49 -0800 Subject: [PATCH 6/9] fix docs build Created using spr 1.3.4 --- llvm/docs/LangRef.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index fece4ac7f127d..51c60ecf5ac65 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -937,8 +937,8 @@ IFuncs, like as aliases, don't create any new data or func. They are just a new symbol that is resolved at runtime by calling a resolver function. On ELF platforms, IFuncs are resolved by the dynamic linker at load time. On -Mach-O platforms, they are lowered in terms of ``.symbol_resolver``s, which -lazily resolve the callee the first time they are called. +Mach-O platforms, they are lowered in terms of ``.symbol_resolver`` functions, +which lazily resolve the callee the first time they are called. IFunc may have an optional :ref:`linkage type <linkage>` and an optional :ref:`visibility style <visibility>`. >From a9cc93c37fba9e40df0d983f1990908cc31dfe76 Mon Sep 17 00:00:00 2001 From: Jon Roelofs <jonathan_roel...@apple.com> Date: Wed, 29 Nov 2023 15:19:14 -0800 Subject: [PATCH 7/9] align data pointer Created using spr 1.3.4 --- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 1 + llvm/lib/Target/X86/X86AsmPrinter.cpp | 1 + llvm/test/CodeGen/AArch64/ifunc-asm.ll | 1 + llvm/test/CodeGen/X86/ifunc-asm.ll | 1 + 4 files changed, 4 insertions(+) diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index 1b5b7c556c79f..a3513ca439ac2 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -1905,6 +1905,7 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, // .quad _ifunc.stub_helper EmitLinkage(LazyPointer); + OutStreamer->emitValueToAlignment(Align(8), /*Value=*/0); OutStreamer->emitLabel(LazyPointer); emitVisibility(LazyPointer, GI.getVisibility()); OutStreamer->emitValue(MCSymbolRefExpr::create(StubHelper, OutContext), 8); diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp index b0f4b9d984372..a490d775abd55 100644 --- a/llvm/lib/Target/X86/X86AsmPrinter.cpp +++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp @@ -558,6 +558,7 @@ void X86AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) { // .quad _ifunc.stub_helper EmitLinkage(LazyPointer); + OutStreamer->emitValueToAlignment(Align(8), /*Value=*/0); OutStreamer->emitLabel(LazyPointer); emitVisibility(LazyPointer, GI.getVisibility()); OutStreamer->emitValue(MCSymbolRefExpr::create(StubHelper, OutContext), 8); diff --git a/llvm/test/CodeGen/AArch64/ifunc-asm.ll b/llvm/test/CodeGen/AArch64/ifunc-asm.ll index ede669aa52703..7eff692da83ff 100644 --- a/llvm/test/CodeGen/AArch64/ifunc-asm.ll +++ b/llvm/test/CodeGen/AArch64/ifunc-asm.ll @@ -35,6 +35,7 @@ entry: ; MACHO-MANUAL: .section __DATA,__data ; MACHO-MANUAL-NEXT: .globl _global_ifunc.lazy_pointer +; MACHO-MANUAL-NEXT: .p2align 3, 0x0 ; MACHO-MANUAL-NEXT: _global_ifunc.lazy_pointer: ; MACHO-MANUAL-NEXT: .quad _global_ifunc.stub_helper diff --git a/llvm/test/CodeGen/X86/ifunc-asm.ll b/llvm/test/CodeGen/X86/ifunc-asm.ll index 0f66febbe95b2..39c326bc6b6c3 100644 --- a/llvm/test/CodeGen/X86/ifunc-asm.ll +++ b/llvm/test/CodeGen/X86/ifunc-asm.ll @@ -19,6 +19,7 @@ entry: ; MACHO: .section __DATA,__data ; MACHO-NEXT: .globl _foo_ifunc.lazy_pointer +; MACHO-NEXT: .p2align 3, 0x0 ; MACHO-NEXT: _foo_ifunc.lazy_pointer: ; MACHO-NEXT: .quad _foo_ifunc.stub_helper ; MACHO-NEXT: .section __TEXT,__text,regular,pure_instructions >From 6fe823c600230e92eb790dcca58a5f41e1e7900e Mon Sep 17 00:00:00 2001 From: Jon Roelofs <jonathan_roel...@apple.com> Date: Fri, 1 Dec 2023 13:08:03 -0800 Subject: [PATCH 8/9] drop the non-manual .symbol_resolver lowering Created using spr 1.3.4 --- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 220 ++++++------------ llvm/test/CodeGen/AArch64/ifunc-asm.ll | 126 +++++----- 2 files changed, 128 insertions(+), 218 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index a3513ca439ac2..f19dcb5be1722 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -71,27 +71,6 @@ using namespace llvm; namespace { -enum class IFuncLowering { - SymbolResolverIfSupported, - SymbolResolverAlways, - SymbolResolverNever -}; - -static cl::opt<IFuncLowering> PreferredIFuncLowering( - "arm64-darwin-ifunc-symbol_resolver", - cl::init(IFuncLowering::SymbolResolverNever), - cl::desc("Pick the lowering for ifuncs on darwin platforms"), cl::Hidden, - cl::values( - clEnumValN( - IFuncLowering::SymbolResolverIfSupported, "if_supported", - "Use .symbol_resolver's when known to be supported by the linker."), - clEnumValN(IFuncLowering::SymbolResolverAlways, "always", - "Always use .symbol_resolvers. NOTE: this might not be " - "supported by the linker in all cases."), - clEnumValN(IFuncLowering::SymbolResolverNever, "never", - "Use a manual lowering, doing what the linker would have " - "done, but in the compiler."))); - class AArch64AsmPrinter : public AsmPrinter { AArch64MCInstLower MCInstLowering; FaultMaps FM; @@ -224,9 +203,6 @@ class AArch64AsmPrinter : public AsmPrinter { } void emitGlobalIFunc(Module &M, const GlobalIFunc &GI) override; - - void emitLinkerSymbolResolver(Module &M, const GlobalIFunc &GI); - void emitManualSymbolResolver(Module &M, const GlobalIFunc &GI); }; } // end anonymous namespace @@ -1838,51 +1814,26 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, TmpInst); } -void AArch64AsmPrinter::emitLinkerSymbolResolver(Module &M, - const GlobalIFunc &GI) { - OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection()); - - MCSymbol *Name = getSymbol(&GI); - - // NOTE: non-global .symbol_resolvers are not yet supported by Darwin linkers - - if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) - OutStreamer->emitSymbolAttribute(Name, MCSA_Global); - else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) - OutStreamer->emitSymbolAttribute(Name, MCSA_WeakReference); - else - assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); - - OutStreamer->emitCodeAlignment(Align(4), STI); - OutStreamer->emitLabel(Name); - OutStreamer->emitSymbolAttribute(Name, MCSA_SymbolResolver); - emitVisibility(Name, GI.getVisibility()); +void AArch64AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) { + if (!TM.getTargetTriple().isOSBinFormatMachO()) + return AsmPrinter::emitGlobalIFunc(M, GI); - // ld-prime does not seem to support aliases of symbol resolvers, so we have - // to tail call the resolver manually. - OutStreamer->emitInstruction( - MCInstBuilder(AArch64::B) - .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), - *STI); -} + // On Darwin platforms, emit a manually-constructed .symbol_resolver that + // implements the symbol resolution duties of the IFunc. + // + // Normally, this would be handled by linker magic, but unfortunately there are + // a few limitations in ld64 and ld-prime's implementation of .symbol_resolver + // that mean we can't always use them: + // + // * resolvers cannot be the target of an alias + // * resolvers cannot have private linkage + // * resolvers cannot have linkonce linkage + // * resolvers cannot appear in executables + // * resolvers cannot appear in bundles + // + // This works around that by emitting a close approximation of what the linker + // would have done. -/// \brief Emit a manually-constructed .symbol_resolver that implements the -/// symbol resolution duties of the IFunc. -/// -/// Normally, this would be handled by linker magic, but unfortunately there are -/// a few limitations in ld64 and ld-prime's implementation of .symbol_resolver -/// that mean we can't always use them: -/// -/// * resolvers cannot be the target of an alias -/// * resolvers cannot have private linkage -/// * resolvers cannot have linkonce linkage -/// * resolvers cannot appear in executables -/// * resolvers cannot appear in bundles -/// -/// This works around that by emitting a close approximation of what the linker -/// would have done. -void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, - const GlobalIFunc &GI) { auto EmitLinkage = [&](MCSymbol *Sym) { if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) OutStreamer->emitSymbolAttribute(Sym, MCSA_Global); @@ -1964,33 +1915,35 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, .addReg(AArch64::X16), *STI); + // These stub helpers are only ever called once, so here we're optimizing for + // minimum size by using the pre-indexed store variants, which saves a few + // bytes of instructions to bump & restore sp. + // _ifunc.stub_helper: - // stp fp, lr, [sp, #-16] - // sub fp, sp, 16 - // stp x1, x0, [sp, #-32] - // stp x3, x2, [sp, #-48] - // stp x5, x4, [sp, #-64] - // stp x7, x6, [sp, #-80] - // stp d1, d0, [sp, #-96] - // stp d3, d2, [sp, #-112] - // stp d5, d4, [sp, #-128] - // stp d7, d6, [sp, #-144] - // sub sp, sp, 144 + // stp fp, lr, [sp, #-16]! + // mov fp, sp + // stp x1, x0, [sp, #-16]! + // stp x3, x2, [sp, #-16]! + // stp x5, x4, [sp, #-16]! + // stp x7, x6, [sp, #-16]! + // stp d1, d0, [sp, #-16]! + // stp d3, d2, [sp, #-16]! + // stp d5, d4, [sp, #-16]! + // stp d7, d6, [sp, #-16]! // bl _resolver // adrp x16, lazy_pointer@GOTPAGE // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] // str x0, [x16] // mov x16, x0 - // add sp, sp, 144 - // ldp d7, d6, [sp, #-144] - // ldp d5, d4, [sp, #-128] - // ldp d3, d2, [sp, #-112] - // ldp d1, d0, [sp, #-96] - // ldp x7, x6, [sp, #-80] - // ldp x5, x4, [sp, #-64] - // ldp x3, x2, [sp, #-48] - // ldp x1, x0, [sp, #-32] - // ldp fp, lr, [sp, #-16] + // ldp d7, d6, [sp], #16 + // ldp d5, d4, [sp], #16 + // ldp d3, d2, [sp], #16 + // ldp d1, d0, [sp], #16 + // ldp x7, x6, [sp], #16 + // ldp x5, x4, [sp], #16 + // ldp x3, x2, [sp], #16 + // ldp x1, x0, [sp], #16 + // ldp fp, lr, [sp], #16 // br x16 EmitLinkage(StubHelper); @@ -1998,43 +1951,39 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, OutStreamer->emitLabel(StubHelper); emitVisibility(StubHelper, GI.getVisibility()); - OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) + .addReg(AArch64::SP) .addReg(AArch64::FP) .addReg(AArch64::LR) .addReg(AArch64::SP) .addImm(-2), *STI); - OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBXri) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) .addReg(AArch64::FP) .addReg(AArch64::SP) - .addImm(16) + .addImm(0) .addImm(0), *STI); - for (int I = 0; I != 8; I += 2) - OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi) - .addReg(AArch64::X1 + I) - .addReg(AArch64::X0 + I) + for (int I = 0; I != 4; ++I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) .addReg(AArch64::SP) - .addImm(-4 - I), + .addReg(AArch64::X1 + 2 * I) + .addReg(AArch64::X0 + 2 * I) + .addReg(AArch64::SP) + .addImm(-2), *STI); - for (int I = 0; I != 8; I += 2) - OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDi) - .addReg(AArch64::D1 + I) - .addReg(AArch64::D0 + I) + for (int I = 0; I != 4; ++I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre) + .addReg(AArch64::SP) + .addReg(AArch64::D1 + 2 * I) + .addReg(AArch64::D0 + 2 * I) .addReg(AArch64::SP) - .addImm(-12 - I), + .addImm(-2), *STI); - OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBXri) - .addReg(AArch64::SP) - .addReg(AArch64::SP) - .addImm(144) - .addImm(0), - *STI); - OutStreamer->emitInstruction( MCInstBuilder(AArch64::BL) .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), @@ -2081,34 +2030,30 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, .addImm(0), *STI); - OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) - .addReg(AArch64::SP) - .addReg(AArch64::SP) - .addImm(144) - .addImm(0), - *STI); - - for (int I = 6; I != -2; I -= 2) - OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDi) - .addReg(AArch64::D1 + I) - .addReg(AArch64::D0 + I) + for (int I = 3; I != -1; --I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost) + .addReg(AArch64::SP) + .addReg(AArch64::D1 + 2 * I) + .addReg(AArch64::D0 + 2 * I) .addReg(AArch64::SP) - .addImm(-12 - I), + .addImm(2), *STI); - for (int I = 6; I != -2; I -= 2) - OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXi) - .addReg(AArch64::X1 + I) - .addReg(AArch64::X0 + I) + for (int I = 3; I != -1; --I) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) .addReg(AArch64::SP) - .addImm(-4 - I), + .addReg(AArch64::X1 + 2 * I) + .addReg(AArch64::X0 + 2 * I) + .addReg(AArch64::SP) + .addImm(2), *STI); - OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXi) + OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) + .addReg(AArch64::SP) .addReg(AArch64::FP) .addReg(AArch64::LR) .addReg(AArch64::SP) - .addImm(-2), + .addImm(2), *STI); OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() @@ -2118,27 +2063,6 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M, *STI); } -void AArch64AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) { - if (!TM.getTargetTriple().isOSBinFormatMachO()) - return AsmPrinter::emitGlobalIFunc(M, GI); - - switch (PreferredIFuncLowering) { - case IFuncLowering::SymbolResolverAlways: - return emitLinkerSymbolResolver(M, GI); - case IFuncLowering::SymbolResolverNever: - return emitManualSymbolResolver(M, GI); - case IFuncLowering::SymbolResolverIfSupported: - if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective()) - return emitLinkerSymbolResolver(M, GI); - else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage()) - // NOTE: non-global .symbol_resolvers are not yet supported by Darwin - // linkers - return emitManualSymbolResolver(M, GI); - else - assert(GI.hasLocalLinkage() && "Invalid ifunc linkage"); - } -} - // Force static initialization. extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() { RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget()); diff --git a/llvm/test/CodeGen/AArch64/ifunc-asm.ll b/llvm/test/CodeGen/AArch64/ifunc-asm.ll index 7eff692da83ff..18c57b577333e 100644 --- a/llvm/test/CodeGen/AArch64/ifunc-asm.ll +++ b/llvm/test/CodeGen/AArch64/ifunc-asm.ll @@ -1,86 +1,72 @@ ; RUN: llc -mtriple=arm64-unknown-linux-gnu %s -filetype=asm -o - | FileCheck %s --check-prefixes=ELF -; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=always | FileCheck %s --check-prefixes=MACHO,MACHO-LINKER -; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=if_supported | FileCheck %s --check-prefixes=MACHO,MACHO-DEFAULT -; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=never | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL -; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL -; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -global-isel | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL +; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - | FileCheck %s --check-prefix=MACHO +; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -global-isel | FileCheck %s --check-prefix=MACHO define internal ptr @the_resolver() { entry: ret ptr null } -; ELF: .type the_resolver,@function -; ELF-NEXT: the_resolver: +; ELF: .type the_resolver,@function +; ELF-NEXT: the_resolver: -; MACHO: .p2align 2 -; MACHO-NEXT: _the_resolver +; MACHO: .p2align 2 +; MACHO-NEXT: _the_resolver: @global_ifunc = ifunc i32 (i32), ptr @the_resolver -; ELF: .globl global_ifunc -; ELF-NEXT: .type global_ifunc,@gnu_indirect_function -; ELF-NEXT: .set global_ifunc, the_resolver +; ELF: .globl global_ifunc +; ELF-NEXT: .type global_ifunc,@gnu_indirect_function +; ELF-NEXT: .set global_ifunc, the_resolver -; MACHO-LINKER: .globl _global_ifunc -; MACHO-LINKER-NEXT: .p2align 2 -; MACHO-LINKER-NEXT: _global_ifunc: -; MACHO-LINKER-NEXT: .symbol_resolver _global_ifunc -; MACHO-LINKER-NEXT: b _the_resolver +; MACHO: .section __DATA,__data +; MACHO-NEXT: .globl _global_ifunc.lazy_pointer +; MACHO-NEXT: .p2align 3, 0x0 +; MACHO-NEXT: _global_ifunc.lazy_pointer: +; MACHO-NEXT: .quad _global_ifunc.stub_helper -; MACHO-DEFAULT: .globl _global_ifunc -; MACHO-DEFAULT-NEXT: .p2align 2 -; MACHO-DEFAULT-NEXT: _global_ifunc: -; MACHO-DEFAULT-NEXT: .symbol_resolver _global_ifunc -; MACHO-DEFAULT-NEXT: b _the_resolver - -; MACHO-MANUAL: .section __DATA,__data -; MACHO-MANUAL-NEXT: .globl _global_ifunc.lazy_pointer -; MACHO-MANUAL-NEXT: .p2align 3, 0x0 -; MACHO-MANUAL-NEXT: _global_ifunc.lazy_pointer: -; MACHO-MANUAL-NEXT: .quad _global_ifunc.stub_helper - -; MACHO-MANUAL: .section __TEXT,__text,regular,pure_instructions -; MACHO-MANUAL-NEXT: .globl _global_ifunc -; MACHO-MANUAL-NEXT: .p2align 2 -; MACHO-MANUAL-NEXT: _global_ifunc: -; MACHO-MANUAL-NEXT: adrp x16, _global_ifunc.lazy_pointer@GOTPAGE -; MACHO-MANUAL-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer@GOTPAGEOFF] -; MACHO-MANUAL-NEXT: ldr x16, [x16] -; MACHO-MANUAL-NEXT: br x16 -; MACHO-MANUAL-NEXT: .globl _global_ifunc.stub_helper -; MACHO-MANUAL-NEXT: .p2align 2 -; MACHO-MANUAL-NEXT: _global_ifunc.stub_helper: -; MACHO-MANUAL-NEXT: stp x29, x30, [sp, #-16] -; MACHO-MANUAL-NEXT: sub x29, sp, #16 -; MACHO-MANUAL-NEXT: stp x1, x0, [sp, #-32] -; MACHO-MANUAL-NEXT: stp x3, x2, [sp, #-48] -; MACHO-MANUAL-NEXT: stp x5, x4, [sp, #-64] -; MACHO-MANUAL-NEXT: stp x7, x6, [sp, #-80] -; MACHO-MANUAL-NEXT: stp d1, d0, [sp, #-96] -; MACHO-MANUAL-NEXT: stp d3, d2, [sp, #-112] -; MACHO-MANUAL-NEXT: stp d5, d4, [sp, #-128] -; MACHO-MANUAL-NEXT: stp d7, d6, [sp, #-144] -; MACHO-MANUAL-NEXT: sub sp, sp, #144 -; MACHO-MANUAL-NEXT: bl _the_resolver -; MACHO-MANUAL-NEXT: adrp x16, _global_ifunc.lazy_pointer@GOTPAGE -; MACHO-MANUAL-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer@GOTPAGEOFF] -; MACHO-MANUAL-NEXT: str x0, [x16] -; MACHO-MANUAL-NEXT: add x16, x0, #0 -; MACHO-MANUAL-NEXT: add sp, sp, #144 -; MACHO-MANUAL-NEXT: ldp d7, d6, [sp, #-144] -; MACHO-MANUAL-NEXT: ldp d5, d4, [sp, #-128] -; MACHO-MANUAL-NEXT: ldp d3, d2, [sp, #-112] -; MACHO-MANUAL-NEXT: ldp d1, d0, [sp, #-96] -; MACHO-MANUAL-NEXT: ldp x7, x6, [sp, #-80] -; MACHO-MANUAL-NEXT: ldp x5, x4, [sp, #-64] -; MACHO-MANUAL-NEXT: ldp x3, x2, [sp, #-48] -; MACHO-MANUAL-NEXT: ldp x1, x0, [sp, #-32] -; MACHO-MANUAL-NEXT: ldp x29, x30, [sp, #-16] -; MACHO-MANUAL-NEXT: br x16 +; MACHO: .section __TEXT,__text,regular,pure_instructions +; MACHO-NEXT: .globl _global_ifunc +; MACHO-NEXT: .p2align 2 +; MACHO-NEXT: _global_ifunc: +; MACHO-NEXT: adrp x16, _global_ifunc.lazy_pointer@GOTPAGE +; MACHO-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer@GOTPAGEOFF] +; MACHO-NEXT: ldr x16, [x16] +; MACHO-NEXT: br x16 +; MACHO-NEXT: .globl _global_ifunc.stub_helper +; MACHO-NEXT: .p2align 2 +; MACHO-NEXT: _global_ifunc.stub_helper: +; MACHO-NEXT: stp x29, x30, [sp, #-16]! +; MACHO-NEXT: mov x29, sp +; MACHO-NEXT: stp x1, x0, [sp, #-16]! +; MACHO-NEXT: stp x3, x2, [sp, #-16]! +; MACHO-NEXT: stp x5, x4, [sp, #-16]! +; MACHO-NEXT: stp x7, x6, [sp, #-16]! +; MACHO-NEXT: stp d1, d0, [sp, #-16]! +; MACHO-NEXT: stp d3, d2, [sp, #-16]! +; MACHO-NEXT: stp d5, d4, [sp, #-16]! +; MACHO-NEXT: stp d7, d6, [sp, #-16]! +; MACHO-NEXT: bl _the_resolver +; MACHO-NEXT: adrp x16, _global_ifunc.lazy_pointer@GOTPAGE +; MACHO-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer@GOTPAGEOFF] +; MACHO-NEXT: str x0, [x16] +; MACHO-NEXT: add x16, x0, #0 +; MACHO-NEXT: ldp d7, d6, [sp], #16 +; MACHO-NEXT: ldp d5, d4, [sp], #16 +; MACHO-NEXT: ldp d3, d2, [sp], #16 +; MACHO-NEXT: ldp d1, d0, [sp], #16 +; MACHO-NEXT: ldp x7, x6, [sp], #16 +; MACHO-NEXT: ldp x5, x4, [sp], #16 +; MACHO-NEXT: ldp x3, x2, [sp], #16 +; MACHO-NEXT: ldp x1, x0, [sp], #16 +; MACHO-NEXT: ldp x29, x30, [sp], #16 +; MACHO-NEXT: br x16 @weak_ifunc = weak ifunc i32 (i32), ptr @the_resolver ; ELF: .type weak_ifunc,@gnu_indirect_function -; MACHO-LINKER: .symbol_resolver _weak_ifunc -; MACHO-MANUAL: _weak_ifunc.stub_helper: -; MACHO-DEFEAULT: _weak_ifunc.stub_helper: \ No newline at end of file +; MACHO: .weak_reference _weak_ifunc.lazy_pointer +; MACHO: _weak_ifunc.lazy_pointer: +; MACHO: .weak_reference _weak_ifunc +; MACHO: _weak_ifunc: +; MACHO: .weak_reference _weak_ifunc.stub_helper +; MACHO: _weak_ifunc.stub_helper: \ No newline at end of file >From 72853bb876a884f2c9af03a858cc3b86d09b09bc Mon Sep 17 00:00:00 2001 From: Jon Roelofs <jonathan_roel...@apple.com> Date: Fri, 1 Dec 2023 13:32:40 -0800 Subject: [PATCH 9/9] fix formatting Created using spr 1.3.4 --- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index f19dcb5be1722..3aca166786128 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -1821,9 +1821,9 @@ void AArch64AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) { // On Darwin platforms, emit a manually-constructed .symbol_resolver that // implements the symbol resolution duties of the IFunc. // - // Normally, this would be handled by linker magic, but unfortunately there are - // a few limitations in ld64 and ld-prime's implementation of .symbol_resolver - // that mean we can't always use them: + // Normally, this would be handled by linker magic, but unfortunately there + // are a few limitations in ld64 and ld-prime's implementation of + // .symbol_resolver that mean we can't always use them: // // * resolvers cannot be the target of an alias // * resolvers cannot have private linkage _______________________________________________ lldb-commits mailing list lldb-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits