https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/236647
>From 96840fdd1e4bd766ab4528b1793bc465ffc89eb6 Mon Sep 17 00:00:00 2001 From: Anshil Gandhi <[email protected]> Date: Wed, 28 Oct 2026 23:21:23 -0400 Subject: [PATCH 1/2] [SandboxVec][LoadStoreVec] Support mixed-type load vectorization Remove the restriction that every load in a top-level load chain must have the vector's element type. A chain of mixed types such as {i32, float} or {double, i32, i32} is loaded as a vector of integers (as chosen by VecUtils::getCombinedVectorTypeFor()), each original load is unpacked from the lanes it occupies, and then reinterpreted back to its own type with Reinterpret::reinterpretValue(). Chains involving a non-integral pointer are left alone, since such a pointer cannot be recovered from an integer lane. Co-Authored-By: Claude Opus 5.5 <[email protected]> --- .../SandboxVectorizer/Passes/LoadStoreVec.h | 9 +- .../SandboxVectorizer/Passes/LoadStoreVec.cpp | 36 +++-- .../load_store_vec_mixed_type_loads.ll | 140 ++++++++++++++++++ 3 files changed, 172 insertions(+), 13 deletions(-) create mode 100644 llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec_mixed_type_loads.ll diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h index d2eccb7e23309f..a62cc5a345f318 100644 --- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h +++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h @@ -91,8 +91,9 @@ class LLVM_ABI LoadStoreVec final : public RegionPass { /// nullopt otherwise. std::optional<Type *> canVectorize(BndlRef<Instruction *> Bndl); - /// Builds a single vector load out of \p Loads. \returns the new load, - /// or nullptr if \p Loads are not a vectorizable. + /// Builds a single vector load out of \p Loads. A chain of mixed types is + /// loaded as a vector of integers. \returns the new load, or nullptr if + /// \p Loads are not vectorizable. LoadInst *createVectorLoad(BndlRef<Instruction *> Loads); /// Packs \p Constants into one constant vector. The element type is chosen @@ -108,7 +109,9 @@ class LLVM_ABI LoadStoreVec final : public RegionPass { /// loads. \returns true on success. bool vectorizeStores(BndlRef<Instruction *> Stores, Region &Rgn); - /// Vectorizes \p Loads into a single load. \return the packed load. + /// Vectorizes \p Loads into a single load and unpacks each original load + /// from it, casting the lanes back to the load's type if needed. + /// \return the packed load. LoadInst *vectorizeLoads(BndlRef<Instruction *> Loads, Region &Rgn); public: diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp index 7247036fc264f5..fb82f1bda87b14 100644 --- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp +++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp @@ -63,6 +63,13 @@ bool LoadStoreVec::acceptOrRevert() { return true; } +/// \returns the type of the lanes of \p LaneTy that hold a value of type \p Ty +/// in a packed vector: \p LaneTy itself, or <N x LaneTy> if \p Ty is wider. +static Type *getLanesTypeFor(Type *Ty, Type *LaneTy, const DataLayout &DL) { + unsigned NumLanes = Utils::getNumBits(Ty, DL) / Utils::getNumBits(LaneTy, DL); + return NumLanes > 1 ? VecUtils::getWideType(LaneTy, NumLanes) : LaneTy; +} + LoadInst *LoadStoreVec::createVectorLoad(BndlRef<Instruction *> Loads) { if (!canVectorize(Loads)) return nullptr; @@ -321,10 +328,12 @@ LoadInst *LoadStoreVec::vectorizeLoads(BndlRef<Instruction *> Loads, if (!VecTy) return nullptr; - // TODO: Support mixed-type top-level load chains. + // Each load is unpacked from the lanes it occupies in the vector and, for a + // chain of mixed types, reinterpreted back as the type of the load. Type *VecElemTy = cast<FixedVectorType>(*VecTy)->getElementType(); - if (!all_of(Loads, [VecElemTy](Instruction *I) { - return VecUtils::getElementType(I->getType()) == VecElemTy; + if (any_of(Loads, [this, VecElemTy](Instruction *I) { + return !Reinterpret::canReinterpret( + getLanesTypeFor(I->getType(), VecElemTy, *DL), I->getType(), *DL); })) return nullptr; @@ -336,14 +345,21 @@ LoadInst *LoadStoreVec::vectorizeLoads(BndlRef<Instruction *> Loads, return nullptr; } + // Lanes are counted in units of LaneTy, so a load wider than LaneTy spans + // several lanes. BasicBlock::iterator WhereIt = std::next(VecLoad->getIterator()); - for (auto [Lane, OrigV] : VecUtils::enumerateLanes(Loads)) { - auto *OrigLoad = cast<LoadInst>(OrigV); - if (OrigLoad->hasNUses(0)) - continue; - Value *Unpacked = - VecUtils::unpack(VecLoad, OrigLoad->getType(), Lane, WhereIt); - OrigLoad->replaceAllUsesWith(Unpacked); + unsigned Lane = 0; + for (Instruction *I : Loads) { + auto *OrigLoad = cast<LoadInst>(I); + Type *VecTy = getLanesTypeFor(OrigLoad->getType(), VecElemTy, *DL); + unsigned NumLanes = VecUtils::getNumLanes(VecTy); + if (!OrigLoad->hasNUses(0)) { + Value *Unpacked = VecUtils::unpack(VecLoad, VecTy, Lane, WhereIt); + Unpacked = Reinterpret::reinterpretValue(Unpacked, OrigLoad->getType(), + WhereIt, *Ctx, *DL); + OrigLoad->replaceAllUsesWith(Unpacked); + } + Lane += NumLanes; } DeadInstrMorgue.collectPotentiallyDeadInstrs(Loads); diff --git a/llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec_mixed_type_loads.ll b/llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec_mixed_type_loads.ll new file mode 100644 index 00000000000000..75077983922ce4 --- /dev/null +++ b/llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec_mixed_type_loads.ll @@ -0,0 +1,140 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -passes=sandbox-vectorizer -sbvec-passes="seed-collection(loads,stores,enable-diff-types)<load-store-vec>" -sbvec-vec-reg-bits=1024 %s -S | FileCheck %s + +target datalayout = "e-p:64:64-ni:4" + +; A load chain of mixed types is loaded as a vector of integers and each lane +; is cast back to the original type. + +; int + float: one lane each, unpacked with a bitcast. +define void @mixed_int_float_loads(ptr %ptr) { +; CHECK-LABEL: define void @mixed_int_float_loads( +; CHECK-SAME: ptr [[PTR:%.*]]) { +; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i32 0 +; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i32>, ptr [[PTR0]], align 4, !sandboxvec [[META0:![0-9]+]] +; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x i32> [[VECIINITL]], i32 0, !sandboxvec [[META0]] +; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x i32> [[VECIINITL]], i32 1, !sandboxvec [[META0]] +; CHECK-NEXT: [[VCAST:%.*]] = bitcast i32 [[UNPACK1]] to float, !sandboxvec [[META0]] +; CHECK-NEXT: call void @use_i32_f32(i32 [[UNPACK]], float [[VCAST]]) +; CHECK-NEXT: ret void +; + %ptr0 = getelementptr i32, ptr %ptr, i32 0 + %ptr1 = getelementptr i32, ptr %ptr, i32 1 + %ld0 = load i32, ptr %ptr0 + %ld1 = load float, ptr %ptr1 + call void @use_i32_f32(i32 %ld0, float %ld1) + ret void +} + +; ptr + int: the pointer lane needs an inttoptr. +define void @mixed_ptr_int_loads(ptr %ptr) { +; CHECK-LABEL: define void @mixed_ptr_int_loads( +; CHECK-SAME: ptr [[PTR:%.*]]) { +; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i64, ptr [[PTR]], i32 0 +; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i64>, ptr [[PTR0]], align 8, !sandboxvec [[META1:![0-9]+]] +; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x i64> [[VECIINITL]], i32 0, !sandboxvec [[META1]] +; CHECK-NEXT: [[VCAST:%.*]] = inttoptr i64 [[UNPACK]] to ptr, !sandboxvec [[META1]] +; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x i64> [[VECIINITL]], i32 1, !sandboxvec [[META1]] +; CHECK-NEXT: call void @use_ptr_i64(ptr [[VCAST]], i64 [[UNPACK1]]) +; CHECK-NEXT: ret void +; + %ptr0 = getelementptr i64, ptr %ptr, i32 0 + %ptr1 = getelementptr i64, ptr %ptr, i32 1 + %ld0 = load ptr, ptr %ptr0 + %ld1 = load i64, ptr %ptr1 + call void @use_ptr_i64(ptr %ld0, i64 %ld1) + ret void +} + +; A double spans two i32 lanes, so it is unpacked with a shuffle and a bitcast. +define void @mixed_double_int_loads(ptr %ptr) { +; CHECK-LABEL: define void @mixed_double_int_loads( +; CHECK-SAME: ptr [[PTR:%.*]]) { +; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0 +; CHECK-NEXT: [[VECIINITL:%.*]] = load <4 x i32>, ptr [[PTR0]], align 8, !sandboxvec [[META2:![0-9]+]] +; CHECK-NEXT: [[UNPACK:%.*]] = shufflevector <4 x i32> [[VECIINITL]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>, !sandboxvec [[META2]] +; CHECK-NEXT: [[VCAST:%.*]] = bitcast <2 x i32> [[UNPACK]] to double, !sandboxvec [[META2]] +; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <4 x i32> [[VECIINITL]], i32 2, !sandboxvec [[META2]] +; CHECK-NEXT: [[UNPACK2:%.*]] = extractelement <4 x i32> [[VECIINITL]], i32 3, !sandboxvec [[META2]] +; CHECK-NEXT: call void @use_f64_i32_i32(double [[VCAST]], i32 [[UNPACK1]], i32 [[UNPACK2]]) +; CHECK-NEXT: ret void +; + %ptr0 = getelementptr i8, ptr %ptr, i32 0 + %ptr1 = getelementptr i8, ptr %ptr, i32 8 + %ptr2 = getelementptr i8, ptr %ptr, i32 12 + %ld0 = load double, ptr %ptr0 + %ld1 = load i32, ptr %ptr1 + %ld2 = load i32, ptr %ptr2 + call void @use_f64_i32_i32(double %ld0, i32 %ld1, i32 %ld2) + ret void +} + +; A vector element mixed with a scalar of another type. +define void @mixed_vector_element_loads(ptr %ptr) { +; CHECK-LABEL: define void @mixed_vector_element_loads( +; CHECK-SAME: ptr [[PTR:%.*]]) { +; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0 +; CHECK-NEXT: [[VECIINITL:%.*]] = load <4 x i32>, ptr [[PTR0]], align 8, !sandboxvec [[META3:![0-9]+]] +; CHECK-NEXT: [[UNPACK:%.*]] = shufflevector <4 x i32> [[VECIINITL]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>, !sandboxvec [[META3]] +; CHECK-NEXT: [[VCAST:%.*]] = bitcast <2 x i32> [[UNPACK]] to <2 x float>, !sandboxvec [[META3]] +; CHECK-NEXT: [[UNPACK1:%.*]] = shufflevector <4 x i32> [[VECIINITL]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>, !sandboxvec [[META3]] +; CHECK-NEXT: [[VCAST2:%.*]] = bitcast <2 x i32> [[UNPACK1]] to i64, !sandboxvec [[META3]] +; CHECK-NEXT: call void @use_v2f32_i64(<2 x float> [[VCAST]], i64 [[VCAST2]]) +; CHECK-NEXT: ret void +; + %ptr0 = getelementptr i8, ptr %ptr, i32 0 + %ptr1 = getelementptr i8, ptr %ptr, i32 8 + %ld0 = load <2 x float>, ptr %ptr0 + %ld1 = load i64, ptr %ptr1 + call void @use_v2f32_i64(<2 x float> %ld0, i64 %ld1) + ret void +} + +; Loads with no uses still merge, and emit no casts. +define void @mixed_loads_no_uses(ptr %ptr) { +; CHECK-LABEL: define void @mixed_loads_no_uses( +; CHECK-SAME: ptr [[PTR:%.*]]) { +; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i32 0 +; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i32>, ptr [[PTR0]], align 4, !sandboxvec [[META4:![0-9]+]] +; CHECK-NEXT: ret void +; + %ptr0 = getelementptr i32, ptr %ptr, i32 0 + %ptr1 = getelementptr i32, ptr %ptr, i32 1 + %ld0 = load i32, ptr %ptr0 + %ld1 = load float, ptr %ptr1 + ret void +} + +; Non-integral pointers have no stable integer representation, so this chain is +; left alone. +define void @mixed_non_integral_ptr_loads(ptr %ptr) { +; CHECK-LABEL: define void @mixed_non_integral_ptr_loads( +; CHECK-SAME: ptr [[PTR:%.*]]) { +; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i64, ptr [[PTR]], i32 0 +; CHECK-NEXT: [[PTR1:%.*]] = getelementptr i64, ptr [[PTR]], i32 1 +; CHECK-NEXT: [[LD0:%.*]] = load ptr addrspace(4), ptr [[PTR0]], align 8, !sandboxvec [[META5:![0-9]+]] +; CHECK-NEXT: [[LD1:%.*]] = load i64, ptr [[PTR1]], align 4, !sandboxvec [[META5]] +; CHECK-NEXT: call void @use_p4_i64(ptr addrspace(4) [[LD0]], i64 [[LD1]]) +; CHECK-NEXT: ret void +; + %ptr0 = getelementptr i64, ptr %ptr, i32 0 + %ptr1 = getelementptr i64, ptr %ptr, i32 1 + %ld0 = load ptr addrspace(4), ptr %ptr0 + %ld1 = load i64, ptr %ptr1 + call void @use_p4_i64(ptr addrspace(4) %ld0, i64 %ld1) + ret void +} + +declare void @use_i32_f32(i32, float) +declare void @use_ptr_i64(ptr, i64) +declare void @use_f64_i32_i32(double, i32, i32) +declare void @use_v2f32_i64(<2 x float>, i64) +declare void @use_p4_i64(ptr addrspace(4), i64) +;. +; CHECK: [[META0]] = distinct !{!"sandboxregion"} +; CHECK: [[META1]] = distinct !{!"sandboxregion"} +; CHECK: [[META2]] = distinct !{!"sandboxregion"} +; CHECK: [[META3]] = distinct !{!"sandboxregion"} +; CHECK: [[META4]] = distinct !{!"sandboxregion"} +; CHECK: [[META5]] = distinct !{!"sandboxregion"} +;. >From 0c3f48ee0e570c366fb1328af8139c778e9e4c0f Mon Sep 17 00:00:00 2001 From: Anshil Gandhi <[email protected]> Date: Wed, 4 Nov 2026 02:35:50 -0600 Subject: [PATCH 2/2] [SandboxVec][LoadStoreVec] Add Reinterpret::reinterpretValue Mixed-type load unpacking is the first caller. Insert the ptrtoint / bitcast / inttoptr chain before the unpack point, and return nullptr when a non-integral pointer is involved. --- .../Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h | 10 +++++++++- .../SandboxVectorizer/Passes/LoadStoreVec.cpp | 11 +++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h index a62cc5a345f318..74ce6ff4355320 100644 --- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h +++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h @@ -36,7 +36,7 @@ class Instruction; class Type; class Constant; -/// Reinterprets the bits of a constant as another type of the same size, e.g., +/// Reinterprets the bits of a value as another type of the same size, e.g., /// float to i32, ptr to i64, or double to <2 x i32>. class Reinterpret { /// A single cast in the chain built by getCastChain(). @@ -61,6 +61,14 @@ class Reinterpret { !VecUtils::isNonIntegralPtr(DestTy, DL); } + /// Reinterprets \p V as \p DestTy, which must have the same size in \p DL, + /// inserting casts before \p WhereIt. Casts of constants fold, so nothing is + /// inserted for those. \returns nullptr if a non-integral pointer is + /// involved. + LLVM_ABI static Value *reinterpretValue(Value *V, Type *DestTy, + BasicBlock::iterator WhereIt, + Context &Ctx, const DataLayout &DL); + /// Reinterprets \p C as \p DestTy, e.g., float 1.0 to i32 0x3f800000, or a /// null pointer to i64 0. \returns nullptr if a non-integral pointer is /// involved. diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp index fb82f1bda87b14..1b122de1208cfc 100644 --- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp +++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp @@ -120,6 +120,17 @@ Reinterpret::getCastChain(Type *SrcTy, Type *DestTy, const DataLayout &DL) { return Chain; } +Value *Reinterpret::reinterpretValue(Value *V, Type *DestTy, + BasicBlock::iterator WhereIt, Context &Ctx, + const DataLayout &DL) { + if (!canReinterpret(V->getType(), DestTy, DL)) + return nullptr; + Value *NewV = V; + for (auto [Opc, Ty] : getCastChain(V->getType(), DestTy, DL)) + NewV = CastInst::create(Ty, Opc, NewV, WhereIt, Ctx, "VCast"); + return NewV; +} + Constant *Reinterpret::reinterpretConstant(Constant *C, Type *DestTy, const DataLayout &DL) { if (!canReinterpret(C->getType(), DestTy, DL)) _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
