llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-aarch64 Author: llvmbot <details> <summary>Changes</summary> Backport 234ce03692ede13ffd2fcb35570d801c1e332814 Requested by: @<!-- -->jroelofs --- Full diff: https://github.com/llvm/llvm-project/pull/220313.diff 2 Files Affected: - (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+18-10) - (added) llvm/test/CodeGen/AArch64/sme-za-tailcall-fpdiff-align.ll (+79) ``````````diff diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index d134e7f911f83..64ec28ebf3e36 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9447,13 +9447,15 @@ SDValue AArch64TargetLowering::LowerFormalArguments( unsigned StackArgSize = CCInfo.getStackSize(); bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt; if (DoesCalleeRestoreStack(CallConv, TailCallOpt)) { + const Align StackAlign = Subtarget->getFrameLowering()->getStackAlign(); + // This is a non-standard ABI so by fiat I say we're allowed to make full - // use of the stack area to be popped, which must be aligned to 16 bytes in - // any case: - StackArgSize = alignTo(StackArgSize, 16); + // use of the stack area to be popped, which must be aligned to the stack + // alignment in any case: + StackArgSize = alignTo(StackArgSize, StackAlign); // If we're expected to restore the stack (e.g. fastcc) then we'll be adding - // a multiple of 16. + // a multiple of the stack alignment. FuncInfo->setArgumentStackToRestore(StackArgSize); // This realignment carries over to the available bytes below. Our own @@ -10286,19 +10288,24 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, // caller will deallocate the entire stack and the callee still expects its // arguments to begin at SP+0. Completely unused for non-tail calls. int FPDiff = 0; + const Align StackAlign = Subtarget->getFrameLowering()->getStackAlign(); if (IsTailCall && !IsSibCall) { unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea(); - // Since callee will pop argument stack as a tail call, we must keep the - // popped size 16-byte aligned. - NumBytes = alignTo(NumBytes, 16); - // FPDiff will be negative if this tail call requires more space than we // would automatically have in our incoming argument space. Positive if we // can actually shrink the stack. FPDiff = NumReusableBytes - NumBytes; + // Since callee will pop the argument stack as a tail call, we must keep the + // popped size aligned to the stack alignment. Either or both of NumBytes + // and NumReusableBytes may not have been aligned, so we further increase by + // the amount needed to keep FPDiff aligned, and therefore preserve the + // required alignment going into the callee. + FPDiff -= offsetToAlignment(FPDiff, StackAlign); + NumBytes += offsetToAlignment(FPDiff, StackAlign); + // Update the required reserved area if this is the tail call requiring the // most argument stack space. if (FPDiff < 0 && FuncInfo->getTailCallReservedStack() < (unsigned)-FPDiff) @@ -10856,8 +10863,9 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, MF.getFunction().getParent()->getModuleFlag("import-call-optimization")) DAG.addCalledGlobal(Chain.getNode(), CalledGlobal, OpFlags); - uint64_t CalleePopBytes = - DoesCalleeRestoreStack(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : 0; + uint64_t CalleePopBytes = DoesCalleeRestoreStack(CallConv, TailCallOpt) + ? alignTo(NumBytes, StackAlign) + : 0; Chain = DAG.getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, DL); InGlue = Chain.getValue(1); diff --git a/llvm/test/CodeGen/AArch64/sme-za-tailcall-fpdiff-align.ll b/llvm/test/CodeGen/AArch64/sme-za-tailcall-fpdiff-align.ll new file mode 100644 index 0000000000000..9c449b12f2127 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sme-za-tailcall-fpdiff-align.ll @@ -0,0 +1,79 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc < %s -mtriple=arm64-apple-ios -mattr=+sme -o - | FileCheck %s +; RUN: llc < %s -mtriple=arm64-apple-ios -mattr=+sme -global-isel-abort=2 -o - 2>&1 | FileCheck %s --check-prefix=CHECK,GISEL + +; GlobalISel doesn't implement SME yet, but once it does, we should be mindful of this case. +; GISEL: warning: Instruction selection used fallback path for caller_more_args +; GISEL: warning: Instruction selection used fallback path for caller_same_args + +declare void @external_func() "aarch64_inout_za" + +; ZA state being live for both caller and callee forces the non-sibcall tail +; call fast path. FPDiff = (caller's incoming argument stack) - (callee's +; outgoing argument stack) can come out either positive or negative, and in +; either case its magnitude must be rounded away from zero to a multiple of +; 16 to keep the stack pointer 16-byte aligned at all times. +declare void @callee_fewer_args(i64, i64, i64, i64, i64, i64, i64, i64) "aarch64_inout_za" +define void @caller_more_args(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) "aarch64_inout_za" uwtable { +; CHECK-LABEL: caller_more_args: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset w30, -8 +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: bl _external_func +; CHECK-NEXT: mov w0, #1 ; =0x1 +; CHECK-NEXT: mov w1, #2 ; =0x2 +; CHECK-NEXT: mov w2, #3 ; =0x3 +; CHECK-NEXT: mov w3, #4 ; =0x4 +; CHECK-NEXT: mov w4, #5 ; =0x5 +; CHECK-NEXT: mov w5, #6 ; =0x6 +; CHECK-NEXT: mov w6, #7 ; =0x7 +; CHECK-NEXT: mov w7, #8 ; =0x8 +; CHECK-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: add sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset -16 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 +; CHECK-NEXT: b _callee_fewer_args +entry: + tail call void @external_func() + tail call void @callee_fewer_args(i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8) "aarch64_inout_za" + ret void +} + +; Likewise for a callee that has the same amount of argument stack as the +; caller, we must round up the amount of used stack space in order to keep +; FPDiff a multiple of 16, thus keeping the stack aligned for the callee. +declare void @callee_same_args(i64, i64, i64, i64, i64, i64, i64, i64, i64) "aarch64_inout_za" +define void @caller_same_args(i64, i64, i64, i64, i64, i64, i64, i64, i64) "aarch64_inout_za" uwtable { +; CHECK-LABEL: caller_same_args: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset w30, -8 +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: bl _external_func +; CHECK-NEXT: mov w8, #9 ; =0x9 +; CHECK-NEXT: mov w0, #1 ; =0x1 +; CHECK-NEXT: mov w1, #2 ; =0x2 +; CHECK-NEXT: str x8, [sp, #16] +; CHECK-NEXT: mov w2, #3 ; =0x3 +; CHECK-NEXT: mov w3, #4 ; =0x4 +; CHECK-NEXT: mov w4, #5 ; =0x5 +; CHECK-NEXT: mov w5, #6 ; =0x6 +; CHECK-NEXT: mov w6, #7 ; =0x7 +; CHECK-NEXT: mov w7, #8 ; =0x8 +; CHECK-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w30 +; CHECK-NEXT: .cfi_restore w29 +; CHECK-NEXT: b _callee_same_args +entry: + tail call void @external_func() + tail call void @callee_same_args(i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9) "aarch64_inout_za" + ret void +} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GISEL: {{.*}} `````````` </details> https://github.com/llvm/llvm-project/pull/220313 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
