https://github.com/pedroclobo updated https://github.com/llvm/llvm-project/pull/237786
>From 670419924ff7aba7eb1d94d4379f8fc5e1d496c4 Mon Sep 17 00:00:00 2001 From: Pedro Lobo <[email protected]> Date: Sat, 31 Oct 2026 16:46:00 +0000 Subject: [PATCH 1/2] [GVN] Pre-commit tests for per-bit `poison` and provenance when forwarding a load --- llvm/test/Transforms/GVN/load-poison.ll | 424 ++++++++++++++++++++++++ 1 file changed, 424 insertions(+) create mode 100644 llvm/test/Transforms/GVN/load-poison.ll diff --git a/llvm/test/Transforms/GVN/load-poison.ll b/llvm/test/Transforms/GVN/load-poison.ll new file mode 100644 index 00000000000000..23d90be46bca9b --- /dev/null +++ b/llvm/test/Transforms/GVN/load-poison.ll @@ -0,0 +1,424 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt < %s -passes=gvn -S | FileCheck %s +; RUN: opt < %s -passes='gvn<memoryssa>' -S | FileCheck %s + +declare void @use(...) memory(none) + +define <2 x i64> @pr99268(ptr %p) { +; CHECK-LABEL: define <2 x i64> @pr99268( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load <2 x i64>, ptr [[P]], align 16 +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[V]] to b128 +; CHECK-NEXT: [[TMP2:%.*]] = bitextract b64, b128 [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP3:%.*]] = bitcast b64 [[TMP2]] to <2 x i32> +; CHECK-NEXT: [[S:%.*]] = sext <2 x i32> [[TMP3]] to <2 x i64> +; CHECK-NEXT: [[R:%.*]] = sub <2 x i64> [[V]], [[S]] +; CHECK-NEXT: ret <2 x i64> [[R]] +; + %v = load <2 x i64>, ptr %p + %l = load <2 x i32>, ptr %p + %s = sext <2 x i32> %l to <2 x i64> + %r = sub <2 x i64> %v, %s + ret <2 x i64> %r +} + +define i8 @load_i32_to_i8(ptr %p) { +; CHECK-LABEL: define i8 @load_i32_to_i8( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(i32 [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[V]] to i8 +; CHECK-NEXT: ret i8 [[TMP1]] +; + %v = load i32, ptr %p + call void (...) @use(i32 %v) + %x = load i8, ptr %p + ret i8 %x +} + +define i32 @load_ptr_to_i32(ptr %p) { +; CHECK-LABEL: define i32 @load_ptr_to_i32( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load ptr, ptr [[P]], align 8 +; CHECK-NEXT: call void (...) @use(ptr [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32 +; CHECK-NEXT: ret i32 [[TMP2]] +; + %v = load ptr, ptr %p + call void (...) @use(ptr %v) + %x = load i32, ptr %p + ret i32 %x +} + +define i32 @load_v4i32_to_i32_straddle(ptr %p) { +; CHECK-LABEL: define i32 @load_v4i32_to_i32_straddle( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr [[P]], align 16 +; CHECK-NEXT: call void (...) @use(<4 x i32> [[V]]) +; CHECK-NEXT: [[Q:%.*]] = getelementptr i8, ptr [[P]], i64 2 +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[V]] to b128 +; CHECK-NEXT: [[TMP2:%.*]] = bitextract i32, b128 [[TMP1]], i32 16 +; CHECK-NEXT: ret i32 [[TMP2]] +; + %v = load <4 x i32>, ptr %p + call void (...) @use(<4 x i32> %v) + %q = getelementptr i8, ptr %p, i64 2 + %x = load i32, ptr %q, align 1 + ret i32 %x +} + +define i8 @load_v2i12_to_i8(ptr %p) { +; CHECK-LABEL: define i8 @load_v2i12_to_i8( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load <2 x i12>, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(<2 x i12> [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i12> [[V]] to b24 +; CHECK-NEXT: [[TMP2:%.*]] = bitextract i8, b24 [[TMP1]], i32 0 +; CHECK-NEXT: ret i8 [[TMP2]] +; + %v = load <2 x i12>, ptr %p + call void (...) @use(<2 x i12> %v) + %x = load i8, ptr %p + ret i8 %x +} + +define i8 @load_float_to_i8(ptr %p) { +; CHECK-LABEL: define i8 @load_float_to_i8( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(float [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = bitcast float [[V]] to i32 +; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8 +; CHECK-NEXT: ret i8 [[TMP2]] +; + %v = load float, ptr %p + call void (...) @use(float %v) + %x = load i8, ptr %p + ret i8 %x +} + +define b32 @load_i32_to_b32(ptr %p) { +; CHECK-LABEL: define b32 @load_i32_to_b32( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(i32 [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[V]] to b32 +; CHECK-NEXT: ret b32 [[TMP1]] +; + %v = load i32, ptr %p + call void (...) @use(i32 %v) + %x = load b32, ptr %p + ret b32 %x +} + +define b32 @load_i32_noundef_to_b32(ptr %p) { +; CHECK-LABEL: define b32 @load_i32_noundef_to_b32( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4, !noundef [[META0:![0-9]+]] +; CHECK-NEXT: call void (...) @use(i32 [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[V]] to b32 +; CHECK-NEXT: ret b32 [[TMP1]] +; + %v = load i32, ptr %p, !noundef !{} + call void (...) @use(i32 %v) + %x = load b32, ptr %p + ret b32 %x +} + +define <vscale x 4 x i32> @load_nxv2i64_to_nxv4i32(ptr %p) { +; CHECK-LABEL: define <vscale x 4 x i32> @load_nxv2i64_to_nxv4i32( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 +; CHECK-NEXT: call void (...) @use(<vscale x 2 x i64> [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <vscale x 2 x i64> [[V]] to <vscale x 4 x i32> +; CHECK-NEXT: ret <vscale x 4 x i32> [[TMP1]] +; + %v = load <vscale x 2 x i64>, ptr %p + call void (...) @use(<vscale x 2 x i64> %v) + %x = load <vscale x 4 x i32>, ptr %p + ret <vscale x 4 x i32> %x +} + +define ptr @load_i64_to_ptr(ptr %p) { +; CHECK-LABEL: define ptr @load_i64_to_ptr( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(i64 [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[V]] to ptr +; CHECK-NEXT: ret ptr [[TMP1]] +; + %v = load i64, ptr %p + call void (...) @use(i64 %v) + %x = load ptr, ptr %p + ret ptr %x +} + +define ptr addrspace(1) @load_ptr_to_ptr1(ptr %p) { +; CHECK-LABEL: define ptr addrspace(1) @load_ptr_to_ptr1( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load ptr, ptr [[P]], align 8 +; CHECK-NEXT: call void (...) @use(ptr [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr addrspace(1) +; CHECK-NEXT: ret ptr addrspace(1) [[TMP2]] +; + %v = load ptr, ptr %p + call void (...) @use(ptr %v) + %x = load ptr addrspace(1), ptr %p + ret ptr addrspace(1) %x +} + +define i8 @load_volatile_i32_to_i8(ptr %p) { +; CHECK-LABEL: define i8 @load_volatile_i32_to_i8( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load volatile i32, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(i32 [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[V]] to i8 +; CHECK-NEXT: ret i8 [[TMP1]] +; + %v = load volatile i32, ptr %p + call void (...) @use(i32 %v) + %x = load i8, ptr %p + ret i8 %x +} + +define i8 @load_i32_to_i16_i8(ptr %p) { +; CHECK-LABEL: define i8 @load_i32_to_i16_i8( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(i32 [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[V]] to i16 +; CHECK-NEXT: [[Q:%.*]] = getelementptr i8, ptr [[P]], i64 2 +; CHECK-NEXT: [[TMP2:%.*]] = lshr i32 [[V]], 16 +; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8 +; CHECK-NEXT: [[T:%.*]] = trunc i16 [[TMP1]] to i8 +; CHECK-NEXT: [[R:%.*]] = add i8 [[T]], [[TMP3]] +; CHECK-NEXT: ret i8 [[R]] +; + %v = load i32, ptr %p + call void (...) @use(i32 %v) + %x = load i16, ptr %p + %q = getelementptr i8, ptr %p, i64 2 + %y = load i8, ptr %q + %t = trunc i16 %x to i8 + %r = add i8 %t, %y + ret i8 %r +} + +define i8 @load_i32_to_i8_nonlocal(ptr %p, i1 %c) { +; CHECK-LABEL: define i8 @load_i32_to_i8_nonlocal( +; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br i1 [[C]], label %[[A:.*]], label %[[B:.*]] +; CHECK: [[A]]: +; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(i32 [[V1]]) +; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[V1]] to i8 +; CHECK-NEXT: br label %[[MERGE:.*]] +; CHECK: [[B]]: +; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(i32 [[V2]]) +; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[V2]] to i8 +; CHECK-NEXT: br label %[[MERGE]] +; CHECK: [[MERGE]]: +; CHECK-NEXT: [[X:%.*]] = phi i8 [ [[TMP1]], %[[B]] ], [ [[TMP0]], %[[A]] ] +; CHECK-NEXT: ret i8 [[X]] +; +entry: + br i1 %c, label %a, label %b + +a: + %v1 = load i32, ptr %p + call void (...) @use(i32 %v1) + br label %merge + +b: + %v2 = load i32, ptr %p + call void (...) @use(i32 %v2) + br label %merge + +merge: + %x = load i8, ptr %p + ret i8 %x +} + +define i8 @load_i32_to_i8_pre(ptr %p, i1 %c) { +; CHECK-LABEL: define i8 @load_i32_to_i8_pre( +; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br i1 [[C]], label %[[A:.*]], label %[[B:.*]] +; CHECK: [[A]]: +; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(i32 [[V]]) +; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[V]] to i8 +; CHECK-NEXT: br label %[[MERGE:.*]] +; CHECK: [[B]]: +; CHECK-NEXT: call void (...) @use() +; CHECK-NEXT: [[X_PRE:%.*]] = load i8, ptr [[P]], align 1 +; CHECK-NEXT: br label %[[MERGE]] +; CHECK: [[MERGE]]: +; CHECK-NEXT: [[X:%.*]] = phi i8 [ [[X_PRE]], %[[B]] ], [ [[TMP0]], %[[A]] ] +; CHECK-NEXT: ret i8 [[X]] +; +entry: + br i1 %c, label %a, label %b + +a: + %v = load i32, ptr %p + call void (...) @use(i32 %v) + br label %merge + +b: + call void (...) @use() + br label %merge + +merge: + %x = load i8, ptr %p + ret i8 %x +} + +; The lanes of %v lie inside %x. +define i32 @neg_load_v4i32_to_i32(ptr %p) { +; CHECK-LABEL: define i32 @neg_load_v4i32_to_i32( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr [[P]], align 16 +; CHECK-NEXT: call void (...) @use(<4 x i32> [[V]]) +; CHECK-NEXT: [[Q:%.*]] = getelementptr i8, ptr [[P]], i64 4 +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[V]] to b128 +; CHECK-NEXT: [[TMP2:%.*]] = bitextract i32, b128 [[TMP1]], i32 32 +; CHECK-NEXT: ret i32 [[TMP2]] +; + %v = load <4 x i32>, ptr %p + call void (...) @use(<4 x i32> %v) + %q = getelementptr i8, ptr %p, i64 4 + %x = load i32, ptr %q + ret i32 %x +} + +; The lanes of %v lie inside the lanes of %x. +define <2 x i64> @neg_load_v4i32_to_v2i64(ptr %p) { +; CHECK-LABEL: define <2 x i64> @neg_load_v4i32_to_v2i64( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr [[P]], align 16 +; CHECK-NEXT: call void (...) @use(<4 x i32> [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[V]] to <2 x i64> +; CHECK-NEXT: ret <2 x i64> [[TMP1]] +; + %v = load <4 x i32>, ptr %p + call void (...) @use(<4 x i32> %v) + %x = load <2 x i64>, ptr %p + ret <2 x i64> %x +} + +; A pointer of the same type keeps provenance. +define ptr @neg_load_v2ptr_to_ptr(ptr %p) { +; CHECK-LABEL: define ptr @neg_load_v2ptr_to_ptr( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load <2 x ptr>, ptr [[P]], align 16 +; CHECK-NEXT: call void (...) @use(<2 x ptr> [[V]]) +; CHECK-NEXT: [[Q:%.*]] = getelementptr i8, ptr [[P]], i64 8 +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x ptr> [[V]] to b128 +; CHECK-NEXT: [[TMP2:%.*]] = bitextract ptr, b128 [[TMP1]], i32 64 +; CHECK-NEXT: ret ptr [[TMP2]] +; + %v = load <2 x ptr>, ptr %p + call void (...) @use(<2 x ptr> %v) + %q = getelementptr i8, ptr %p, i64 8 + %x = load ptr, ptr %q + ret ptr %x +} + +; %v is never poison. +define i8 @neg_load_i32_noundef_to_i8(ptr %p) { +; CHECK-LABEL: define i8 @neg_load_i32_noundef_to_i8( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4, !noundef [[META0]] +; CHECK-NEXT: call void (...) @use(i32 [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[V]] to i8 +; CHECK-NEXT: ret i8 [[TMP1]] +; + %v = load i32, ptr %p, !noundef !{} + call void (...) @use(i32 %v) + %x = load i8, ptr %p + ret i8 %x +} + +; %v is already a byte. +define i8 @neg_load_b32_to_i8(ptr %p) { +; CHECK-LABEL: define i8 @neg_load_b32_to_i8( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load b32, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(b32 [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = bitextract i8, b32 [[V]], i32 0 +; CHECK-NEXT: ret i8 [[TMP1]] +; + %v = load b32, ptr %p + call void (...) @use(b32 %v) + %x = load i8, ptr %p + ret i8 %x +} + +; %x reads a whole lane of %v. +define i32 @neg_load_volatile_float_to_i32(ptr %p) { +; CHECK-LABEL: define i32 @neg_load_volatile_float_to_i32( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load volatile float, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(float [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = bitcast float [[V]] to i32 +; CHECK-NEXT: ret i32 [[TMP1]] +; + %v = load volatile float, ptr %p + call void (...) @use(float %v) + %x = load i32, ptr %p + ret i32 %x +} + +; A volatile load can't change from floating-point to integer. +define i8 @neg_load_volatile_float_to_i8(ptr %p) { +; CHECK-LABEL: define i8 @neg_load_volatile_float_to_i8( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V:%.*]] = load volatile float, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(float [[V]]) +; CHECK-NEXT: [[TMP1:%.*]] = bitcast float [[V]] to i32 +; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8 +; CHECK-NEXT: ret i8 [[TMP2]] +; + %v = load volatile float, ptr %p + call void (...) @use(float %v) + %x = load i8, ptr %p + ret i8 %x +} + +; GVN can't erase %v while it visits the block. +define void @neg_load_i32_to_i8_loop(ptr %p, i1 %c) { +; CHECK-LABEL: define void @neg_load_i32_to_i8_loop( +; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[X_PRE:%.*]] = load i8, ptr [[P]], align 1 +; CHECK-NEXT: [[V_PRE:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[X:%.*]] = phi i8 [ [[TMP0:%.*]], %[[LOOP]] ], [ [[X_PRE]], %[[ENTRY]] ] +; CHECK-NEXT: call void (...) @use(i8 [[X]], i32 [[V_PRE]]) +; CHECK-NEXT: [[TMP0]] = trunc i32 [[V_PRE]] to i8 +; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %x = load i8, ptr %p + %v = load i32, ptr %p + call void (...) @use(i8 %x, i32 %v) + br i1 %c, label %loop, label %exit + +exit: + ret void +} + +;. +; CHECK: [[META0]] = !{} +;. >From 0f9ccd46eee19bd54bd7811aac92b352b14d9045 Mon Sep 17 00:00:00 2001 From: Pedro Lobo <[email protected]> Date: Sat, 31 Oct 2026 17:01:17 +0000 Subject: [PATCH 2/2] [GVN] Preserve per-bit `poison` and provenance when forwarding a load --- llvm/lib/Transforms/Scalar/GVN.cpp | 103 +++++++++++++++- llvm/test/Transforms/GVN/PRE/atomic.ll | 5 +- llvm/test/Transforms/GVN/PRE/rle.ll | 53 ++++----- llvm/test/Transforms/GVN/invariant.group.ll | 5 +- llvm/test/Transforms/GVN/load-poison.ll | 111 ++++++++++-------- llvm/test/Transforms/GVN/metadata.ll | 10 +- .../Transforms/GVN/rle-coerced-noalias.ll | 14 +-- 7 files changed, 206 insertions(+), 95 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp index 297d3cbc1549b0..89ef5efea534b8 100644 --- a/llvm/lib/Transforms/Scalar/GVN.cpp +++ b/llvm/lib/Transforms/Scalar/GVN.cpp @@ -414,6 +414,16 @@ class llvm::GVNPassImpl { AvailValInBlkVect &ValuesPerBlock, UnavailBlkVect &UnavailableBlocks); + /// Return true if \p DepLoad can be forwarded to \p Load at byte \p Offset + /// exactly, possibly after turning \p DepLoad into a byte load. + bool canForwardLoad(LoadInst *DepLoad, unsigned Offset, LoadInst *Load); + + /// If forwarding \p AV to \p Load loses bits of memory, replace the load it + /// forwards from with a byte load. If \p ValuesPerBlock is given, update its + /// entries that refer to the old load. + void convertToByteLoad(LoadInst *Load, AvailableValue &AV, + AvailValInBlkVect *ValuesPerBlock = nullptr); + /// Given a critical edge from Pred to LoadBB, find a load instruction /// which is identical to Load from another successor of Pred. LoadInst *findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB, @@ -1641,6 +1651,86 @@ GVNPassImpl::analyzeSelectAvailability(LoadInst *Load, SelectInst *Sel, return AvailableValue::getSelect(Sel, V1, V2); } +/// Return true if forwarding \p DepLoad to \p Load at byte \p Offset can lose +/// bits that \p Load reads from memory, so \p DepLoad must become a byte load. +static bool needsByteLoad(LoadInst *DepLoad, unsigned Offset, LoadInst *Load, + AssumptionCache *AC, DominatorTree *DT) { + Type *DepTy = DepLoad->getType(); + Type *LoadTy = Load->getType(); + if (DepTy->isByteOrByteVectorTy()) + return false; + // Only a byte keeps every bit of memory, including pointer bits. + if (LoadTy->isByteOrByteVectorTy()) + return true; + // Only a pointer of the same type keeps the provenance of a pointer. + if (LoadTy->isPtrOrPtrVectorTy() && + DepTy->getScalarType() != LoadTy->getScalarType()) + return true; + if (isGuaranteedNotToBePoison(DepLoad, AC, Load, DT)) + return false; + // Memory is poison per bit, while vectors are poison per lane and other types + // are poison per value. Each lane of Load must cover whole lanes of DepLoad. + const DataLayout &DL = DepLoad->getDataLayout(); + uint64_t DepBits = + DL.getTypeSizeInBits(DepTy->getScalarType()).getFixedValue(); + uint64_t LoadBits = + DL.getTypeSizeInBits(LoadTy->getScalarType()).getFixedValue(); + return (Offset * 8) % DepBits != 0 || LoadBits % DepBits != 0; +} + +bool GVNPassImpl::canForwardLoad(LoadInst *DepLoad, unsigned Offset, + LoadInst *Load) { + if (!needsByteLoad(DepLoad, Offset, Load, AC, DT)) + return true; + // DepLoad must become a byte load. Bytes lower to integers, and a volatile + // load can't change from floating-point to integer. + if (DepLoad->isVolatile() && DepLoad->getType()->isFPOrFPVectorTy()) + return false; + // processBlock may already point to a DepLoad that comes after Load. + return DepLoad->getParent() != Load->getParent() || + DepLoad->comesBefore(Load); +} + +void GVNPassImpl::convertToByteLoad(LoadInst *Load, AvailableValue &AV, + AvailValInBlkVect *ValuesPerBlock) { + if (!AV.isCoercedLoadValue()) + return; + LoadInst *DepLoad = AV.getCoercedLoadValue(); + if (!needsByteLoad(DepLoad, AV.Offset, Load, AC, DT)) + return; + LLVM_DEBUG(dbgs() << "GVN CONVERTED TO BYTE LOAD: " << *DepLoad << '\n'); + + Type *Ty = DepLoad->getType(); + Type *ByteTy = Ty->isPtrOrPtrVectorTy() + ? DepLoad->getDataLayout().getBytePtrType(Ty) + : Type::getByteFromIntType(Ty); + auto *ByteLoad = new LoadInst( + ByteTy, DepLoad->getPointerOperand(), DepLoad->getName() + ".bytes", + DepLoad->getProperties(), DepLoad->getIterator()); + copyMetadataForLoad(*ByteLoad, *DepLoad); + if (MSSAU) { + auto *NewAccess = MSSAU->createMemoryAccessBefore( + ByteLoad, nullptr, MSSAU->getMemorySSA()->getMemoryAccess(DepLoad)); + if (auto *NewDef = dyn_cast<MemoryDef>(NewAccess)) + MSSAU->insertDef(NewDef, /*RenameUses=*/true); + else + MSSAU->insertUse(cast<MemoryUse>(NewAccess), /*RenameUses=*/true); + } + + auto *Cast = new BitCastInst(ByteLoad, Ty, "", DepLoad->getIterator()); + Cast->setDebugLoc(DepLoad->getDebugLoc()); + Cast->takeName(DepLoad); + + ICF->removeUsersOf(DepLoad); + DepLoad->replaceAllUsesWith(Cast); + if (ValuesPerBlock) + replaceValuesPerBlockEntry(*ValuesPerBlock, DepLoad, Cast); + if (uint32_t ValNo = VN.lookup(DepLoad, false)) + LeaderTable.erase(ValNo, DepLoad, DepLoad->getParent()); + removeInstruction(DepLoad); + AV.Val = ByteLoad; +} + std::optional<AvailableValue> GVNPassImpl::analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep, Value *Address) { @@ -1697,7 +1787,7 @@ GVNPassImpl::analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep, if (Offset == -1) Offset = analyzeLoadFromClobberingLoad(LoadType, Address, DepLoad, DL); - if (Offset != -1) + if (Offset != -1 && canForwardLoad(DepLoad, Offset, Load)) return AvailableValue::getLoad(DepLoad, Offset); } } @@ -1761,6 +1851,9 @@ GVNPassImpl::analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep, if (LD->isAtomic() < Load->isAtomic()) return std::nullopt; + if (!canForwardLoad(LD, 0, Load)) + return std::nullopt; + return AvailableValue::getLoad(LD); } @@ -1981,6 +2074,9 @@ void GVNPassImpl::eliminatePartiallyRedundantLoad( } } + for (AvailableValueInBlock &AV : ValuesPerBlock) + convertToByteLoad(Load, AV.AV, &ValuesPerBlock); + // Perform PHI construction. Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, getDominatorTree()); // constructSSAForLoadSet is responsible for combining metadata. @@ -2436,6 +2532,9 @@ bool GVNPassImpl::processNonLocalLoad(LoadInst *Load, if (UnavailableBlocks.empty()) { LLVM_DEBUG(dbgs() << "GVN REMOVING NONLOCAL LOAD: " << *Load << '\n'); + for (AvailableValueInBlock &AV : ValuesPerBlock) + convertToByteLoad(Load, AV.AV, &ValuesPerBlock); + // Perform PHI construction. Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, getDominatorTree()); // constructSSAForLoadSet is responsible for combining metadata. @@ -3147,6 +3246,8 @@ bool GVNPassImpl::processLoad(LoadInst *L) { if (!AV) return false; + convertToByteLoad(L, *AV); + Value *AvailableValue = AV->MaterializeAdjustedValue(L, L); // MaterializeAdjustedValue is responsible for combining metadata. diff --git a/llvm/test/Transforms/GVN/PRE/atomic.ll b/llvm/test/Transforms/GVN/PRE/atomic.ll index ed530bec22e846..fb60ac26004d2b 100644 --- a/llvm/test/Transforms/GVN/PRE/atomic.ll +++ b/llvm/test/Transforms/GVN/PRE/atomic.ll @@ -426,8 +426,9 @@ define i64 @widen1(ptr %P1) { define i64 @narrow(ptr %P1) { ; CHECK-LABEL: define i64 @narrow ; CHECK-SAME: (ptr [[P1:%.*]]) { -; CHECK-NEXT: [[A64:%.*]] = load atomic i64, ptr [[P1]] unordered, align 4 -; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[A64]] to i32 +; CHECK-NEXT: [[A64_BYTES:%.*]] = load atomic b64, ptr [[P1]] unordered, align 4 +; CHECK-NEXT: [[A64:%.*]] = bitcast b64 [[A64_BYTES]] to i64 +; CHECK-NEXT: [[TMP1:%.*]] = bitextract i32, b64 [[A64_BYTES]], i32 0 ; CHECK-NEXT: [[B64:%.*]] = sext i32 [[TMP1]] to i64 ; CHECK-NEXT: [[RES:%.*]] = sub i64 [[A64]], [[B64]] ; CHECK-NEXT: ret i64 [[RES]] diff --git a/llvm/test/Transforms/GVN/PRE/rle.ll b/llvm/test/Transforms/GVN/PRE/rle.ll index 49fe873c0141f1..1e8da7a5eff335 100644 --- a/llvm/test/Transforms/GVN/PRE/rle.ll +++ b/llvm/test/Transforms/GVN/PRE/rle.ll @@ -951,9 +951,9 @@ define i32 @load_load_partial_alias(ptr %P) nounwind ssp { ; LE-LABEL: define i32 @load_load_partial_alias( ; LE-SAME: ptr [[P:%.*]]) #[[ATTR0]] { ; LE-NEXT: [[ENTRY:.*:]] -; LE-NEXT: [[TTMP2:%.*]] = load i32, ptr [[P]], align 4 -; LE-NEXT: [[TMP0:%.*]] = lshr i32 [[TTMP2]], 8 -; LE-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i8 +; LE-NEXT: [[TTMP2_BYTES:%.*]] = load b32, ptr [[P]], align 4 +; LE-NEXT: [[TTMP2:%.*]] = bitcast b32 [[TTMP2_BYTES]] to i32 +; LE-NEXT: [[TMP1:%.*]] = bitextract i8, b32 [[TTMP2_BYTES]], i32 8 ; LE-NEXT: [[CONV:%.*]] = zext i8 [[TMP1]] to i32 ; LE-NEXT: [[ADD:%.*]] = add nsw i32 [[TTMP2]], [[CONV]] ; LE-NEXT: ret i32 [[ADD]] @@ -961,9 +961,9 @@ define i32 @load_load_partial_alias(ptr %P) nounwind ssp { ; BE-LABEL: define i32 @load_load_partial_alias( ; BE-SAME: ptr [[P:%.*]]) #[[ATTR0]] { ; BE-NEXT: [[ENTRY:.*:]] -; BE-NEXT: [[TTMP2:%.*]] = load i32, ptr [[P]], align 4 -; BE-NEXT: [[TMP0:%.*]] = lshr i32 [[TTMP2]], 16 -; BE-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i8 +; BE-NEXT: [[TTMP2_BYTES:%.*]] = load b32, ptr [[P]], align 4 +; BE-NEXT: [[TTMP2:%.*]] = bitcast b32 [[TTMP2_BYTES]] to i32 +; BE-NEXT: [[TMP1:%.*]] = bitextract i8, b32 [[TTMP2_BYTES]], i32 16 ; BE-NEXT: [[CONV:%.*]] = zext i8 [[TMP1]] to i32 ; BE-NEXT: [[ADD:%.*]] = add nsw i32 [[TTMP2]], [[CONV]] ; BE-NEXT: ret i32 [[ADD]] @@ -1139,21 +1139,22 @@ define void @load_load_partial_alias_loop(ptr %P) { ; LE-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 1 ; LE-NEXT: [[V_1:%.*]] = load i8, ptr [[P_1]], align 1 ; LE-NEXT: call void @use.i8(i8 [[V_1]]) -; LE-NEXT: [[V_1_32:%.*]] = load i32, ptr [[P_1]], align 4 +; LE-NEXT: [[V_1_32_BYTES:%.*]] = load b32, ptr [[P_1]], align 4 +; LE-NEXT: [[V_1_32:%.*]] = bitcast b32 [[V_1_32_BYTES]] to i32 ; LE-NEXT: call void @use.i32(i32 [[V_1_32]]) -; LE-NEXT: [[TMP0:%.*]] = trunc i32 [[V_1_32]] to i8 +; LE-NEXT: [[TMP0:%.*]] = bitextract i8, b32 [[V_1_32_BYTES]], i32 0 ; LE-NEXT: br label %[[LOOP:.*]] ; LE: [[LOOP]]: -; LE-NEXT: [[V_I:%.*]] = phi i8 [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[LOOP_LOOP_CRIT_EDGE:.*]] ] +; LE-NEXT: [[V_I:%.*]] = phi i8 [ [[TMP0]], %[[ENTRY]] ], [ [[TMP1:%.*]], %[[LOOP_LOOP_CRIT_EDGE:.*]] ] ; LE-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_INC:%.*]], %[[LOOP_LOOP_CRIT_EDGE]] ] ; LE-NEXT: [[P_I:%.*]] = getelementptr i8, ptr [[P]], i64 [[I]] ; LE-NEXT: call void @use.i8(i8 [[V_I]]) -; LE-NEXT: [[V_I_32:%.*]] = load i32, ptr [[P_I]], align 4 +; LE-NEXT: [[V_I_32_BYTES:%.*]] = load b32, ptr [[P_I]], align 4 +; LE-NEXT: [[V_I_32:%.*]] = bitcast b32 [[V_I_32_BYTES]] to i32 ; LE-NEXT: call void @use.i32(i32 [[V_I_32]]) ; LE-NEXT: [[I_INC]] = add i64 [[I]], 1 ; LE-NEXT: [[CMP:%.*]] = icmp ne i64 [[I_INC]], 64 -; LE-NEXT: [[TMP1:%.*]] = lshr i32 [[V_I_32]], 8 -; LE-NEXT: [[TMP2]] = trunc i32 [[TMP1]] to i8 +; LE-NEXT: [[TMP1]] = bitextract i8, b32 [[V_I_32_BYTES]], i32 8 ; LE-NEXT: br i1 [[CMP]], label %[[LOOP_LOOP_CRIT_EDGE]], label %[[EXIT:.*]] ; LE: [[LOOP_LOOP_CRIT_EDGE]]: ; LE-NEXT: br label %[[LOOP]] @@ -1166,22 +1167,22 @@ define void @load_load_partial_alias_loop(ptr %P) { ; BE-NEXT: [[P_1:%.*]] = getelementptr i8, ptr [[P]], i64 1 ; BE-NEXT: [[V_1:%.*]] = load i8, ptr [[P_1]], align 1 ; BE-NEXT: call void @use.i8(i8 [[V_1]]) -; BE-NEXT: [[V_1_32:%.*]] = load i32, ptr [[P_1]], align 4 +; BE-NEXT: [[V_1_32_BYTES:%.*]] = load b32, ptr [[P_1]], align 4 +; BE-NEXT: [[V_1_32:%.*]] = bitcast b32 [[V_1_32_BYTES]] to i32 ; BE-NEXT: call void @use.i32(i32 [[V_1_32]]) -; BE-NEXT: [[TMP0:%.*]] = lshr i32 [[V_1_32]], 24 -; BE-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i8 +; BE-NEXT: [[TMP0:%.*]] = bitextract i8, b32 [[V_1_32_BYTES]], i32 24 ; BE-NEXT: br label %[[LOOP:.*]] ; BE: [[LOOP]]: -; BE-NEXT: [[V_I:%.*]] = phi i8 [ [[TMP1]], %[[ENTRY]] ], [ [[TMP3:%.*]], %[[LOOP_LOOP_CRIT_EDGE:.*]] ] +; BE-NEXT: [[V_I:%.*]] = phi i8 [ [[TMP0]], %[[ENTRY]] ], [ [[TMP1:%.*]], %[[LOOP_LOOP_CRIT_EDGE:.*]] ] ; BE-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_INC:%.*]], %[[LOOP_LOOP_CRIT_EDGE]] ] ; BE-NEXT: [[P_I:%.*]] = getelementptr i8, ptr [[P]], i64 [[I]] ; BE-NEXT: call void @use.i8(i8 [[V_I]]) -; BE-NEXT: [[V_I_32:%.*]] = load i32, ptr [[P_I]], align 4 +; BE-NEXT: [[V_I_32_BYTES:%.*]] = load b32, ptr [[P_I]], align 4 +; BE-NEXT: [[V_I_32:%.*]] = bitcast b32 [[V_I_32_BYTES]] to i32 ; BE-NEXT: call void @use.i32(i32 [[V_I_32]]) ; BE-NEXT: [[I_INC]] = add i64 [[I]], 1 ; BE-NEXT: [[CMP:%.*]] = icmp ne i64 [[I_INC]], 64 -; BE-NEXT: [[TMP2:%.*]] = lshr i32 [[V_I_32]], 16 -; BE-NEXT: [[TMP3]] = trunc i32 [[TMP2]] to i8 +; BE-NEXT: [[TMP1]] = bitextract i8, b32 [[V_I_32_BYTES]], i32 16 ; BE-NEXT: br i1 [[CMP]], label %[[LOOP_LOOP_CRIT_EDGE]], label %[[EXIT:.*]] ; BE: [[LOOP_LOOP_CRIT_EDGE]]: ; BE-NEXT: br label %[[LOOP]] @@ -1221,13 +1222,12 @@ define void @load_load_partial_alias_atomic(ptr %arg) { ; LE-SAME: ptr [[ARG:%.*]]) { ; LE-NEXT: [[BB:.*]]: ; LE-NEXT: [[TMP2_1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 1 -; LE-NEXT: [[TMP2_3:%.*]] = load i64, ptr [[TMP2_1]], align 4 +; LE-NEXT: [[TMP2_3_BYTES:%.*]] = load b64, ptr [[TMP2_1]], align 4 ; LE-NEXT: [[TMP3_1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 2 -; LE-NEXT: [[TMP0:%.*]] = lshr i64 [[TMP2_3]], 8 -; LE-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i8 +; LE-NEXT: [[TMP0:%.*]] = bitextract i8, b64 [[TMP2_3_BYTES]], i32 8 ; LE-NEXT: br label %[[BB5:.*]] ; LE: [[BB5]]: -; LE-NEXT: [[TMP4_1:%.*]] = phi i8 [ [[TMP4_1_PRE:%.*]], %[[BB5]] ], [ [[TMP1]], %[[BB]] ] +; LE-NEXT: [[TMP4_1:%.*]] = phi i8 [ [[TMP4_1_PRE:%.*]], %[[BB5]] ], [ [[TMP0]], %[[BB]] ] ; LE-NEXT: [[TMP6_1:%.*]] = load atomic i8, ptr @global acquire, align 4 ; LE-NEXT: [[TMP7_1:%.*]] = add i8 [[TMP6_1]], [[TMP4_1]] ; LE-NEXT: store i8 [[TMP7_1]], ptr [[ARG]], align 1 @@ -1238,13 +1238,12 @@ define void @load_load_partial_alias_atomic(ptr %arg) { ; BE-SAME: ptr [[ARG:%.*]]) { ; BE-NEXT: [[BB:.*]]: ; BE-NEXT: [[TMP2_1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 1 -; BE-NEXT: [[TMP2_3:%.*]] = load i64, ptr [[TMP2_1]], align 4 +; BE-NEXT: [[TMP2_3_BYTES:%.*]] = load b64, ptr [[TMP2_1]], align 4 ; BE-NEXT: [[TMP3_1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 2 -; BE-NEXT: [[TMP0:%.*]] = lshr i64 [[TMP2_3]], 48 -; BE-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i8 +; BE-NEXT: [[TMP0:%.*]] = bitextract i8, b64 [[TMP2_3_BYTES]], i32 48 ; BE-NEXT: br label %[[BB5:.*]] ; BE: [[BB5]]: -; BE-NEXT: [[TMP4_1:%.*]] = phi i8 [ [[TMP4_1_PRE:%.*]], %[[BB5]] ], [ [[TMP1]], %[[BB]] ] +; BE-NEXT: [[TMP4_1:%.*]] = phi i8 [ [[TMP4_1_PRE:%.*]], %[[BB5]] ], [ [[TMP0]], %[[BB]] ] ; BE-NEXT: [[TMP6_1:%.*]] = load atomic i8, ptr @global acquire, align 4 ; BE-NEXT: [[TMP7_1:%.*]] = add i8 [[TMP6_1]], [[TMP4_1]] ; BE-NEXT: store i8 [[TMP7_1]], ptr [[ARG]], align 1 diff --git a/llvm/test/Transforms/GVN/invariant.group.ll b/llvm/test/Transforms/GVN/invariant.group.ll index 50dbfe4f3100b4..80cadc5f463118 100644 --- a/llvm/test/Transforms/GVN/invariant.group.ll +++ b/llvm/test/Transforms/GVN/invariant.group.ll @@ -469,11 +469,12 @@ define void @testNotGlobal() { ; CHECK-LABEL: define void @testNotGlobal() { ; CHECK-NEXT: [[A:%.*]] = alloca i8, align 1 ; CHECK-NEXT: call void @foo(ptr [[A]]) -; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[A]], align 1, !invariant.group [[META0]] +; CHECK-NEXT: [[B_BYTES:%.*]] = load b8, ptr [[A]], align 1, !invariant.group [[META0]] +; CHECK-NEXT: [[B:%.*]] = bitcast b8 [[B_BYTES]] to i8 ; CHECK-NEXT: call void @foo2(ptr [[A]], i8 [[B]]) ; CHECK-NEXT: call void @bar(i8 [[B]]) ; CHECK-NEXT: call void @fooBit(ptr [[A]], i1 true) -; CHECK-NEXT: [[TMP1:%.*]] = trunc i8 [[B]] to i1 +; CHECK-NEXT: [[TMP1:%.*]] = bitextract i1, b8 [[B_BYTES]], i32 0 ; CHECK-NEXT: call void @fooBit(ptr [[A]], i1 [[TMP1]]) ; CHECK-NEXT: call void @fooBit(ptr [[A]], i1 [[TMP1]]) ; CHECK-NEXT: ret void diff --git a/llvm/test/Transforms/GVN/load-poison.ll b/llvm/test/Transforms/GVN/load-poison.ll index 23d90be46bca9b..dbb54fcb0bc217 100644 --- a/llvm/test/Transforms/GVN/load-poison.ll +++ b/llvm/test/Transforms/GVN/load-poison.ll @@ -7,8 +7,9 @@ declare void @use(...) memory(none) define <2 x i64> @pr99268(ptr %p) { ; CHECK-LABEL: define <2 x i64> @pr99268( ; CHECK-SAME: ptr [[P:%.*]]) { -; CHECK-NEXT: [[V:%.*]] = load <2 x i64>, ptr [[P]], align 16 -; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[V]] to b128 +; CHECK-NEXT: [[V_BYTES:%.*]] = load <2 x b64>, ptr [[P]], align 16 +; CHECK-NEXT: [[V:%.*]] = bitcast <2 x b64> [[V_BYTES]] to <2 x i64> +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x b64> [[V_BYTES]] to b128 ; CHECK-NEXT: [[TMP2:%.*]] = bitextract b64, b128 [[TMP1]], i32 0 ; CHECK-NEXT: [[TMP3:%.*]] = bitcast b64 [[TMP2]] to <2 x i32> ; CHECK-NEXT: [[S:%.*]] = sext <2 x i32> [[TMP3]] to <2 x i64> @@ -25,9 +26,10 @@ define <2 x i64> @pr99268(ptr %p) { define i8 @load_i32_to_i8(ptr %p) { ; CHECK-LABEL: define i8 @load_i32_to_i8( ; CHECK-SAME: ptr [[P:%.*]]) { -; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: [[V_BYTES:%.*]] = load b32, ptr [[P]], align 4 +; CHECK-NEXT: [[V:%.*]] = bitcast b32 [[V_BYTES]] to i32 ; CHECK-NEXT: call void (...) @use(i32 [[V]]) -; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[V]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = bitextract i8, b32 [[V_BYTES]], i32 0 ; CHECK-NEXT: ret i8 [[TMP1]] ; %v = load i32, ptr %p @@ -39,11 +41,11 @@ define i8 @load_i32_to_i8(ptr %p) { define i32 @load_ptr_to_i32(ptr %p) { ; CHECK-LABEL: define i32 @load_ptr_to_i32( ; CHECK-SAME: ptr [[P:%.*]]) { -; CHECK-NEXT: [[V:%.*]] = load ptr, ptr [[P]], align 8 +; CHECK-NEXT: [[V_BYTES:%.*]] = load b64, ptr [[P]], align 8 +; CHECK-NEXT: [[V:%.*]] = bitcast b64 [[V_BYTES]] to ptr ; CHECK-NEXT: call void (...) @use(ptr [[V]]) -; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64 -; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32 -; CHECK-NEXT: ret i32 [[TMP2]] +; CHECK-NEXT: [[TMP1:%.*]] = bitextract i32, b64 [[V_BYTES]], i32 0 +; CHECK-NEXT: ret i32 [[TMP1]] ; %v = load ptr, ptr %p call void (...) @use(ptr %v) @@ -54,10 +56,11 @@ define i32 @load_ptr_to_i32(ptr %p) { define i32 @load_v4i32_to_i32_straddle(ptr %p) { ; CHECK-LABEL: define i32 @load_v4i32_to_i32_straddle( ; CHECK-SAME: ptr [[P:%.*]]) { -; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr [[P]], align 16 +; CHECK-NEXT: [[V_BYTES:%.*]] = load <4 x b32>, ptr [[P]], align 16 +; CHECK-NEXT: [[V:%.*]] = bitcast <4 x b32> [[V_BYTES]] to <4 x i32> ; CHECK-NEXT: call void (...) @use(<4 x i32> [[V]]) ; CHECK-NEXT: [[Q:%.*]] = getelementptr i8, ptr [[P]], i64 2 -; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[V]] to b128 +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x b32> [[V_BYTES]] to b128 ; CHECK-NEXT: [[TMP2:%.*]] = bitextract i32, b128 [[TMP1]], i32 16 ; CHECK-NEXT: ret i32 [[TMP2]] ; @@ -71,9 +74,10 @@ define i32 @load_v4i32_to_i32_straddle(ptr %p) { define i8 @load_v2i12_to_i8(ptr %p) { ; CHECK-LABEL: define i8 @load_v2i12_to_i8( ; CHECK-SAME: ptr [[P:%.*]]) { -; CHECK-NEXT: [[V:%.*]] = load <2 x i12>, ptr [[P]], align 4 +; CHECK-NEXT: [[V_BYTES:%.*]] = load <2 x b12>, ptr [[P]], align 4 +; CHECK-NEXT: [[V:%.*]] = bitcast <2 x b12> [[V_BYTES]] to <2 x i12> ; CHECK-NEXT: call void (...) @use(<2 x i12> [[V]]) -; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i12> [[V]] to b24 +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x b12> [[V_BYTES]] to b24 ; CHECK-NEXT: [[TMP2:%.*]] = bitextract i8, b24 [[TMP1]], i32 0 ; CHECK-NEXT: ret i8 [[TMP2]] ; @@ -86,11 +90,11 @@ define i8 @load_v2i12_to_i8(ptr %p) { define i8 @load_float_to_i8(ptr %p) { ; CHECK-LABEL: define i8 @load_float_to_i8( ; CHECK-SAME: ptr [[P:%.*]]) { -; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4 +; CHECK-NEXT: [[V_BYTES:%.*]] = load b32, ptr [[P]], align 4 +; CHECK-NEXT: [[V:%.*]] = bitcast b32 [[V_BYTES]] to float ; CHECK-NEXT: call void (...) @use(float [[V]]) -; CHECK-NEXT: [[TMP1:%.*]] = bitcast float [[V]] to i32 -; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8 -; CHECK-NEXT: ret i8 [[TMP2]] +; CHECK-NEXT: [[TMP1:%.*]] = bitextract i8, b32 [[V_BYTES]], i32 0 +; CHECK-NEXT: ret i8 [[TMP1]] ; %v = load float, ptr %p call void (...) @use(float %v) @@ -101,10 +105,10 @@ define i8 @load_float_to_i8(ptr %p) { define b32 @load_i32_to_b32(ptr %p) { ; CHECK-LABEL: define b32 @load_i32_to_b32( ; CHECK-SAME: ptr [[P:%.*]]) { -; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: [[V_BYTES:%.*]] = load b32, ptr [[P]], align 4 +; CHECK-NEXT: [[V:%.*]] = bitcast b32 [[V_BYTES]] to i32 ; CHECK-NEXT: call void (...) @use(i32 [[V]]) -; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[V]] to b32 -; CHECK-NEXT: ret b32 [[TMP1]] +; CHECK-NEXT: ret b32 [[V_BYTES]] ; %v = load i32, ptr %p call void (...) @use(i32 %v) @@ -115,10 +119,10 @@ define b32 @load_i32_to_b32(ptr %p) { define b32 @load_i32_noundef_to_b32(ptr %p) { ; CHECK-LABEL: define b32 @load_i32_noundef_to_b32( ; CHECK-SAME: ptr [[P:%.*]]) { -; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4, !noundef [[META0:![0-9]+]] +; CHECK-NEXT: [[V_BYTES:%.*]] = load b32, ptr [[P]], align 4, !noundef [[META0:![0-9]+]] +; CHECK-NEXT: [[V:%.*]] = bitcast b32 [[V_BYTES]] to i32 ; CHECK-NEXT: call void (...) @use(i32 [[V]]) -; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[V]] to b32 -; CHECK-NEXT: ret b32 [[TMP1]] +; CHECK-NEXT: ret b32 [[V_BYTES]] ; %v = load i32, ptr %p, !noundef !{} call void (...) @use(i32 %v) @@ -129,9 +133,10 @@ define b32 @load_i32_noundef_to_b32(ptr %p) { define <vscale x 4 x i32> @load_nxv2i64_to_nxv4i32(ptr %p) { ; CHECK-LABEL: define <vscale x 4 x i32> @load_nxv2i64_to_nxv4i32( ; CHECK-SAME: ptr [[P:%.*]]) { -; CHECK-NEXT: [[V:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16 +; CHECK-NEXT: [[V_BYTES:%.*]] = load <vscale x 2 x b64>, ptr [[P]], align 16 +; CHECK-NEXT: [[V:%.*]] = bitcast <vscale x 2 x b64> [[V_BYTES]] to <vscale x 2 x i64> ; CHECK-NEXT: call void (...) @use(<vscale x 2 x i64> [[V]]) -; CHECK-NEXT: [[TMP1:%.*]] = bitcast <vscale x 2 x i64> [[V]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <vscale x 2 x b64> [[V_BYTES]] to <vscale x 4 x i32> ; CHECK-NEXT: ret <vscale x 4 x i32> [[TMP1]] ; %v = load <vscale x 2 x i64>, ptr %p @@ -143,9 +148,10 @@ define <vscale x 4 x i32> @load_nxv2i64_to_nxv4i32(ptr %p) { define ptr @load_i64_to_ptr(ptr %p) { ; CHECK-LABEL: define ptr @load_i64_to_ptr( ; CHECK-SAME: ptr [[P:%.*]]) { -; CHECK-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 4 +; CHECK-NEXT: [[V_BYTES:%.*]] = load b64, ptr [[P]], align 4 +; CHECK-NEXT: [[V:%.*]] = bitcast b64 [[V_BYTES]] to i64 ; CHECK-NEXT: call void (...) @use(i64 [[V]]) -; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[V]] to ptr +; CHECK-NEXT: [[TMP1:%.*]] = bitcast b64 [[V_BYTES]] to ptr ; CHECK-NEXT: ret ptr [[TMP1]] ; %v = load i64, ptr %p @@ -157,11 +163,11 @@ define ptr @load_i64_to_ptr(ptr %p) { define ptr addrspace(1) @load_ptr_to_ptr1(ptr %p) { ; CHECK-LABEL: define ptr addrspace(1) @load_ptr_to_ptr1( ; CHECK-SAME: ptr [[P:%.*]]) { -; CHECK-NEXT: [[V:%.*]] = load ptr, ptr [[P]], align 8 +; CHECK-NEXT: [[V_BYTES:%.*]] = load b64, ptr [[P]], align 8 +; CHECK-NEXT: [[V:%.*]] = bitcast b64 [[V_BYTES]] to ptr ; CHECK-NEXT: call void (...) @use(ptr [[V]]) -; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64 -; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr addrspace(1) -; CHECK-NEXT: ret ptr addrspace(1) [[TMP2]] +; CHECK-NEXT: [[TMP1:%.*]] = bitcast b64 [[V_BYTES]] to ptr addrspace(1) +; CHECK-NEXT: ret ptr addrspace(1) [[TMP1]] ; %v = load ptr, ptr %p call void (...) @use(ptr %v) @@ -172,9 +178,10 @@ define ptr addrspace(1) @load_ptr_to_ptr1(ptr %p) { define i8 @load_volatile_i32_to_i8(ptr %p) { ; CHECK-LABEL: define i8 @load_volatile_i32_to_i8( ; CHECK-SAME: ptr [[P:%.*]]) { -; CHECK-NEXT: [[V:%.*]] = load volatile i32, ptr [[P]], align 4 +; CHECK-NEXT: [[V_BYTES:%.*]] = load volatile b32, ptr [[P]], align 4 +; CHECK-NEXT: [[V:%.*]] = bitcast b32 [[V_BYTES]] to i32 ; CHECK-NEXT: call void (...) @use(i32 [[V]]) -; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[V]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = bitextract i8, b32 [[V_BYTES]], i32 0 ; CHECK-NEXT: ret i8 [[TMP1]] ; %v = load volatile i32, ptr %p @@ -186,14 +193,14 @@ define i8 @load_volatile_i32_to_i8(ptr %p) { define i8 @load_i32_to_i16_i8(ptr %p) { ; CHECK-LABEL: define i8 @load_i32_to_i16_i8( ; CHECK-SAME: ptr [[P:%.*]]) { -; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: [[V_BYTES:%.*]] = load b32, ptr [[P]], align 4 +; CHECK-NEXT: [[V:%.*]] = bitcast b32 [[V_BYTES]] to i32 ; CHECK-NEXT: call void (...) @use(i32 [[V]]) -; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[V]] to i16 +; CHECK-NEXT: [[TMP1:%.*]] = bitextract i16, b32 [[V_BYTES]], i32 0 ; CHECK-NEXT: [[Q:%.*]] = getelementptr i8, ptr [[P]], i64 2 -; CHECK-NEXT: [[TMP2:%.*]] = lshr i32 [[V]], 16 -; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8 +; CHECK-NEXT: [[TMP2:%.*]] = bitextract i8, b32 [[V_BYTES]], i32 16 ; CHECK-NEXT: [[T:%.*]] = trunc i16 [[TMP1]] to i8 -; CHECK-NEXT: [[R:%.*]] = add i8 [[T]], [[TMP3]] +; CHECK-NEXT: [[R:%.*]] = add i8 [[T]], [[TMP2]] ; CHECK-NEXT: ret i8 [[R]] ; %v = load i32, ptr %p @@ -212,14 +219,16 @@ define i8 @load_i32_to_i8_nonlocal(ptr %p, i1 %c) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br i1 [[C]], label %[[A:.*]], label %[[B:.*]] ; CHECK: [[A]]: -; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: [[V1_BYTES:%.*]] = load b32, ptr [[P]], align 4 +; CHECK-NEXT: [[V1:%.*]] = bitcast b32 [[V1_BYTES]] to i32 ; CHECK-NEXT: call void (...) @use(i32 [[V1]]) -; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[V1]] to i8 +; CHECK-NEXT: [[TMP0:%.*]] = bitextract i8, b32 [[V1_BYTES]], i32 0 ; CHECK-NEXT: br label %[[MERGE:.*]] ; CHECK: [[B]]: -; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: [[V2_BYTES:%.*]] = load b32, ptr [[P]], align 4 +; CHECK-NEXT: [[V2:%.*]] = bitcast b32 [[V2_BYTES]] to i32 ; CHECK-NEXT: call void (...) @use(i32 [[V2]]) -; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[V2]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = bitextract i8, b32 [[V2_BYTES]], i32 0 ; CHECK-NEXT: br label %[[MERGE]] ; CHECK: [[MERGE]]: ; CHECK-NEXT: [[X:%.*]] = phi i8 [ [[TMP1]], %[[B]] ], [ [[TMP0]], %[[A]] ] @@ -249,9 +258,10 @@ define i8 @load_i32_to_i8_pre(ptr %p, i1 %c) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br i1 [[C]], label %[[A:.*]], label %[[B:.*]] ; CHECK: [[A]]: -; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: [[V_BYTES:%.*]] = load b32, ptr [[P]], align 4 +; CHECK-NEXT: [[V:%.*]] = bitcast b32 [[V_BYTES]] to i32 ; CHECK-NEXT: call void (...) @use(i32 [[V]]) -; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[V]] to i8 +; CHECK-NEXT: [[TMP0:%.*]] = bitextract i8, b32 [[V_BYTES]], i32 0 ; CHECK-NEXT: br label %[[MERGE:.*]] ; CHECK: [[B]]: ; CHECK-NEXT: call void (...) @use() @@ -380,9 +390,8 @@ define i8 @neg_load_volatile_float_to_i8(ptr %p) { ; CHECK-SAME: ptr [[P:%.*]]) { ; CHECK-NEXT: [[V:%.*]] = load volatile float, ptr [[P]], align 4 ; CHECK-NEXT: call void (...) @use(float [[V]]) -; CHECK-NEXT: [[TMP1:%.*]] = bitcast float [[V]] to i32 -; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8 -; CHECK-NEXT: ret i8 [[TMP2]] +; CHECK-NEXT: [[X:%.*]] = load i8, ptr [[P]], align 1 +; CHECK-NEXT: ret i8 [[X]] ; %v = load volatile float, ptr %p call void (...) @use(float %v) @@ -394,14 +403,12 @@ define i8 @neg_load_volatile_float_to_i8(ptr %p) { define void @neg_load_i32_to_i8_loop(ptr %p, i1 %c) { ; CHECK-LABEL: define void @neg_load_i32_to_i8_loop( ; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: [[X_PRE:%.*]] = load i8, ptr [[P]], align 1 -; CHECK-NEXT: [[V_PRE:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: -; CHECK-NEXT: [[X:%.*]] = phi i8 [ [[TMP0:%.*]], %[[LOOP]] ], [ [[X_PRE]], %[[ENTRY]] ] -; CHECK-NEXT: call void (...) @use(i8 [[X]], i32 [[V_PRE]]) -; CHECK-NEXT: [[TMP0]] = trunc i32 [[V_PRE]] to i8 +; CHECK-NEXT: [[X:%.*]] = load i8, ptr [[P]], align 1 +; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: call void (...) @use(i8 [[X]], i32 [[V]]) ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void diff --git a/llvm/test/Transforms/GVN/metadata.ll b/llvm/test/Transforms/GVN/metadata.ll index c2725e99570c6b..0eb0fdd9783490 100644 --- a/llvm/test/Transforms/GVN/metadata.ll +++ b/llvm/test/Transforms/GVN/metadata.ll @@ -253,9 +253,10 @@ define void @load_ptr_dereferenceable_or_null_to_i64(ptr %p) { define void @load_ptr_nonnull_to_i32(ptr %p) { ; CHECK-LABEL: define void @load_ptr_nonnull_to_i32 ; CHECK-SAME: (ptr [[P:%.*]]) { -; CHECK-NEXT: [[VAL:%.*]] = load ptr, ptr [[P]], align 8 +; CHECK-NEXT: [[VAL_BYTES:%.*]] = load b64, ptr [[P]], align 8 +; CHECK-NEXT: [[VAL:%.*]] = bitcast b64 [[VAL_BYTES]] to ptr ; CHECK-NEXT: [[VAL_INT:%.*]] = ptrtoint ptr [[VAL]] to i64 -; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[VAL_INT]] to i32 +; CHECK-NEXT: [[TMP1:%.*]] = bitextract i32, b64 [[VAL_BYTES]], i32 0 ; CHECK-NEXT: call void @use.i64(i64 [[VAL_INT]]) ; CHECK-NEXT: call void @use.i32(i32 [[TMP1]]) ; CHECK-NEXT: ret void @@ -271,8 +272,9 @@ define void @load_ptr_nonnull_to_i32(ptr %p) { define void @load_i64_range_to_i32_range(ptr %p) { ; CHECK-LABEL: define void @load_i64_range_to_i32_range ; CHECK-SAME: (ptr [[P:%.*]]) { -; CHECK-NEXT: [[VAL:%.*]] = load i64, ptr [[P]], align 8 -; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[VAL]] to i32 +; CHECK-NEXT: [[VAL_BYTES:%.*]] = load b64, ptr [[P]], align 8 +; CHECK-NEXT: [[VAL:%.*]] = bitcast b64 [[VAL_BYTES]] to i64 +; CHECK-NEXT: [[TMP1:%.*]] = bitextract i32, b64 [[VAL_BYTES]], i32 0 ; CHECK-NEXT: call void @use.i64(i64 [[VAL]]) ; CHECK-NEXT: call void @use.i32(i32 [[TMP1]]) ; CHECK-NEXT: ret void diff --git a/llvm/test/Transforms/GVN/rle-coerced-noalias.ll b/llvm/test/Transforms/GVN/rle-coerced-noalias.ll index 1ae70ee2fa8d34..9c37a3f15bde8c 100644 --- a/llvm/test/Transforms/GVN/rle-coerced-noalias.ll +++ b/llvm/test/Transforms/GVN/rle-coerced-noalias.ll @@ -9,15 +9,15 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" ; The i8 load at P+1 partially aliases the i32 load at P and is coerced from -; it. The surviving i32 load should keep its !noalias. +; it. The i32 load becomes a byte load, which should keep its !noalias. define i32 @noalias_dropped_local(ptr %p) { ; CHECK-LABEL: define i32 @noalias_dropped_local( ; CHECK-SAME: ptr [[P:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] -; CHECK-NEXT: [[WIDE:%.*]] = load i32, ptr [[P]], align 4, !noalias [[META0:![0-9]+]] +; CHECK-NEXT: [[WIDE_BYTES:%.*]] = load b32, ptr [[P]], align 4, !noalias [[META0:![0-9]+]] +; CHECK-NEXT: [[WIDE:%.*]] = bitcast b32 [[WIDE_BYTES]] to i32 ; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1 -; CHECK-NEXT: [[TMP0:%.*]] = lshr i32 [[WIDE]], 8 -; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = bitextract i8, b32 [[WIDE_BYTES]], i32 8 ; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP1]] to i32 ; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[WIDE]], [[CONV]] ; CHECK-NEXT: ret i32 [[ADD]] @@ -36,10 +36,10 @@ define i32 @alias_scope_dropped_local(ptr %p) { ; CHECK-LABEL: define i32 @alias_scope_dropped_local( ; CHECK-SAME: ptr [[P:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] -; CHECK-NEXT: [[WIDE:%.*]] = load i32, ptr [[P]], align 4, !alias.scope [[META0]] +; CHECK-NEXT: [[WIDE_BYTES:%.*]] = load b32, ptr [[P]], align 4, !alias.scope [[META0]] +; CHECK-NEXT: [[WIDE:%.*]] = bitcast b32 [[WIDE_BYTES]] to i32 ; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1 -; CHECK-NEXT: [[TMP0:%.*]] = lshr i32 [[WIDE]], 8 -; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = bitextract i8, b32 [[WIDE_BYTES]], i32 8 ; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP1]] to i32 ; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[WIDE]], [[CONV]] ; CHECK-NEXT: ret i32 [[ADD]] _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
