https://github.com/paschalis-mpeis updated
https://github.com/llvm/llvm-project/pull/78432
>From fbe30cfa2e9474241be71f334228bb811678d0a2 Mon Sep 17 00:00:00 2001
From: Paschalis Mpeis <paschalis.mp...@arm.com>
Date: Tue, 16 Jan 2024 10:53:09 +0000
Subject: [PATCH 1/5] LAA cannot vectorize lib calls like modf/modff
Functions like modf/modff are math lib calls that set memory write-only
attribute. Given that a target has vectorized mappings, LAA should allow
vectorization.
---
...arch64-veclib-function-calls-linear-ptrs.c | 57 +++++++++++++++++++
1 file changed, 57 insertions(+)
create mode 100644
clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
diff --git a/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
b/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
new file mode 100644
index 0000000000000..a449fac147058
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
@@ -0,0 +1,57 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
UTC_ARGS: --filter "call.*(frexp|modf)" --version 4
+// RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve -O3 -mllvm
-vector-library=ArmPL -mllvm -force-vector-interleave=1 -mllvm
-prefer-predicate-over-epilogue=predicate-dont-vectorize -emit-llvm -S -o - %s
| FileCheck %s
+
+// REQUIRES: aarch64-registered-target
+
+/*
+Testing vectorization of math functions that have the attribute write-only to
+memory set. Given they have vectorized counterparts, they should be able to
+vectorize.
+*/
+
+// The following define is required to access some math functions.
+#define _GNU_SOURCE
+#include <math.h>
+
+// frexp/frexpf have no TLI mappings yet.
+
+// CHECK-LABEL: define dso_local void @frexp_f64(
+// CHECK-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture
noundef writeonly [[OUT1:%.*]], ptr nocapture noundef writeonly [[OUT2:%.*]],
i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK: [[CALL:%.*]] = tail call double @frexp(double noundef
[[TMP0:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR2:[0-9]+]]
+//
+void frexp_f64(double *in, double *out1, int *out2, int N) {
+ for (int i = 0; i < N; ++i)
+ *out1 = frexp(in[i], out2+i);
+}
+
+// CHECK-LABEL: define dso_local void @frexp_f32(
+// CHECK-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture
noundef writeonly [[OUT1:%.*]], ptr nocapture noundef writeonly [[OUT2:%.*]],
i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK: [[CALL:%.*]] = tail call float @frexpf(float noundef
[[TMP0:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR2]]
+//
+void frexp_f32(float *in, float *out1, int *out2, int N) {
+ for (int i = 0; i < N; ++i)
+ *out1 = frexpf(in[i], out2+i);
+}
+
+
+// TODO: LAA must allow vectorization.
+
+// CHECK-LABEL: define dso_local void @modf_f64(
+// CHECK-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture
noundef writeonly [[OUT1:%.*]], ptr nocapture noundef writeonly [[OUT2:%.*]],
i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK: [[CALL:%.*]] = tail call double @modf(double noundef
[[TMP0:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR3:[0-9]+]]
+//
+void modf_f64(double *in, double *out1, double *out2, int N) {
+ for (int i = 0; i < N; ++i)
+ out1[i] = modf(in[i], out2+i);
+}
+
+// TODO: LAA must allow vectorization.
+
+// CHECK-LABEL: define dso_local void @modf_f32(
+// CHECK-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture
noundef writeonly [[OUT1:%.*]], ptr nocapture noundef writeonly [[OUT2:%.*]],
i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK: [[CALL:%.*]] = tail call float @modff(float noundef [[TMP0:%.*]],
ptr noundef [[ADD_PTR:%.*]]) #[[ATTR4:[0-9]+]]
+//
+void modf_f32(float *in, float *out1, float *out2, int N) {
+ for (int i = 0; i < N; ++i)
+ out1[i] = modff(in[i], out2+i);
+}
>From 80a86c8a75877cc1e00e2d72bc4ed424728ec6fc Mon Sep 17 00:00:00 2001
From: Paschalis Mpeis <paschalis.mp...@arm.com>
Date: Wed, 17 Jan 2024 09:44:45 +0000
Subject: [PATCH 2/5] [LV][LAA] Vectorize math lib calls with mem write-only
attribute
Teach LAA to consider safe specific math lib calls which are known to
have set the memory write-only attribute. Those attributes are set to
calls by inferNonMandatoryLibFuncAttrs, in BuildLibCalls.cpp, and the
current ones are modf/modff and frexp/frexpf.
This happens only when the calls are found through TLI to have
vectorized counterparts.
---
...arch64-veclib-function-calls-linear-ptrs.c | 15 ++++++---------
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 19 +++++++++++++++++++
2 files changed, 25 insertions(+), 9 deletions(-)
diff --git a/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
b/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
index a449fac147058..957b3f5cb235d 100644
--- a/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
+++ b/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
@@ -17,7 +17,7 @@ vectorize.
// CHECK-LABEL: define dso_local void @frexp_f64(
// CHECK-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture
noundef writeonly [[OUT1:%.*]], ptr nocapture noundef writeonly [[OUT2:%.*]],
i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK: [[CALL:%.*]] = tail call double @frexp(double noundef
[[TMP0:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR2:[0-9]+]]
+// CHECK: [[CALL:%.*]] = tail call double @frexp(double noundef
[[TMP0:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR5:[0-9]+]]
//
void frexp_f64(double *in, double *out1, int *out2, int N) {
for (int i = 0; i < N; ++i)
@@ -26,30 +26,27 @@ void frexp_f64(double *in, double *out1, int *out2, int N) {
// CHECK-LABEL: define dso_local void @frexp_f32(
// CHECK-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture
noundef writeonly [[OUT1:%.*]], ptr nocapture noundef writeonly [[OUT2:%.*]],
i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK: [[CALL:%.*]] = tail call float @frexpf(float noundef
[[TMP0:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR2]]
+// CHECK: [[CALL:%.*]] = tail call float @frexpf(float noundef
[[TMP0:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR5]]
//
void frexp_f32(float *in, float *out1, int *out2, int N) {
for (int i = 0; i < N; ++i)
*out1 = frexpf(in[i], out2+i);
}
-
-// TODO: LAA must allow vectorization.
-
// CHECK-LABEL: define dso_local void @modf_f64(
// CHECK-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture
noundef writeonly [[OUT1:%.*]], ptr nocapture noundef writeonly [[OUT2:%.*]],
i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK: [[CALL:%.*]] = tail call double @modf(double noundef
[[TMP0:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR3:[0-9]+]]
+// CHECK: [[TMP11:%.*]] = tail call <vscale x 2 x double>
@armpl_svmodf_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], ptr
[[TMP10:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+// CHECK: [[CALL:%.*]] = tail call double @modf(double noundef
[[TMP14:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR6:[0-9]+]]
//
void modf_f64(double *in, double *out1, double *out2, int N) {
for (int i = 0; i < N; ++i)
out1[i] = modf(in[i], out2+i);
}
-// TODO: LAA must allow vectorization.
-
// CHECK-LABEL: define dso_local void @modf_f32(
// CHECK-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture
noundef writeonly [[OUT1:%.*]], ptr nocapture noundef writeonly [[OUT2:%.*]],
i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK: [[CALL:%.*]] = tail call float @modff(float noundef [[TMP0:%.*]],
ptr noundef [[ADD_PTR:%.*]]) #[[ATTR4:[0-9]+]]
+// CHECK: [[TMP11:%.*]] = tail call <vscale x 4 x float>
@armpl_svmodf_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], ptr
[[TMP10:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+// CHECK: [[CALL:%.*]] = tail call float @modff(float noundef
[[TMP14:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR7:[0-9]+]]
//
void modf_f32(float *in, float *out1, float *out2, int N) {
for (int i = 0; i < N; ++i)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index dd6b88fee415a..523d60b3ed15a 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -2309,6 +2309,20 @@ bool LoopAccessInfo::canAnalyzeLoop() {
return true;
}
+/// Returns whether \p I is a known math library call that has memory
write-only
+/// attribute set.
+static bool isMathLibCallMemWriteOnly(const TargetLibraryInfo *TLI,
+ const Instruction &I) {
+ auto *Call = dyn_cast<CallInst>(&I);
+ if (!Call)
+ return false;
+
+ LibFunc Func;
+ TLI->getLibFunc(*Call, Func);
+ return Func == LibFunc::LibFunc_modf || Func == LibFunc::LibFunc_modff ||
+ Func == LibFunc::LibFunc_frexp || Func == LibFunc::LibFunc_frexpf;
+}
+
void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,
const TargetLibraryInfo *TLI,
DominatorTree *DT) {
@@ -2405,6 +2419,11 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo
*LI,
// Save 'store' instructions. Abort if other instructions write to
memory.
if (I.mayWriteToMemory()) {
+ // We can safety handle math functions that have vectorized
+ // counterparts and have the memory write-only attribute set.
+ if (isMathLibCallMemWriteOnly(TLI, I))
+ continue;
+
auto *St = dyn_cast<StoreInst>(&I);
if (!St) {
recordAnalysis("CantVectorizeInstruction", St)
>From a36008867cc147aebfb55e8a1cc15c990ef41f42 Mon Sep 17 00:00:00 2001
From: Paschalis Mpeis <paschalis.mp...@arm.com>
Date: Thu, 18 Jan 2024 14:14:00 +0000
Subject: [PATCH 3/5] Add check for the 'memory(argmem: write)' attribute.
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 523d60b3ed15a..0a57782921b68 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -2309,18 +2309,24 @@ bool LoopAccessInfo::canAnalyzeLoop() {
return true;
}
-/// Returns whether \p I is a known math library call that has memory
write-only
-/// attribute set.
+/// Returns whether \p I is a known math library call that has attribute
+/// 'memory(argmem: write)' set.
static bool isMathLibCallMemWriteOnly(const TargetLibraryInfo *TLI,
const Instruction &I) {
auto *Call = dyn_cast<CallInst>(&I);
if (!Call)
return false;
+ Function *F = Call->getCalledFunction();
+ if (!F->hasFnAttribute(Attribute::AttrKind::Memory))
+ return false;
+
+ auto ME = F->getFnAttribute(Attribute::AttrKind::Memory).getMemoryEffects();
LibFunc Func;
TLI->getLibFunc(*Call, Func);
- return Func == LibFunc::LibFunc_modf || Func == LibFunc::LibFunc_modff ||
- Func == LibFunc::LibFunc_frexp || Func == LibFunc::LibFunc_frexpf;
+ return ME.onlyWritesMemory() && ME.onlyAccessesArgPointees() &&
+ (Func == LibFunc::LibFunc_modf || Func == LibFunc::LibFunc_modff ||
+ Func == LibFunc::LibFunc_frexp || Func == LibFunc::LibFunc_frexpf);
}
void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,
>From 2a91269140aac18f96d60a393cbbe15101530db1 Mon Sep 17 00:00:00 2001
From: Paschalis Mpeis <paschalis.mp...@arm.com>
Date: Fri, 19 Jan 2024 10:20:50 +0000
Subject: [PATCH 4/5] Addressing reviewers
---
.../CodeGen/aarch64-veclib-function-calls-linear-ptrs.c | 2 +-
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 6 +-----
2 files changed, 2 insertions(+), 6 deletions(-)
diff --git a/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
b/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
index 957b3f5cb235d..98085a183f46c 100644
--- a/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
+++ b/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
@@ -1,5 +1,5 @@
// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
UTC_ARGS: --filter "call.*(frexp|modf)" --version 4
-// RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve -O3 -mllvm
-vector-library=ArmPL -mllvm -force-vector-interleave=1 -mllvm
-prefer-predicate-over-epilogue=predicate-dont-vectorize -emit-llvm -S -o - %s
| FileCheck %s
+// RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve -O3 -isystem
%S/../Headers/Inputs/include -mllvm -vector-library=ArmPL -mllvm
-force-vector-interleave=1 -mllvm
-prefer-predicate-over-epilogue=predicate-dont-vectorize -emit-llvm -S -o - %s
| FileCheck %s
// REQUIRES: aarch64-registered-target
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 0a57782921b68..03e096298a7be 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -2317,11 +2317,7 @@ static bool isMathLibCallMemWriteOnly(const
TargetLibraryInfo *TLI,
if (!Call)
return false;
- Function *F = Call->getCalledFunction();
- if (!F->hasFnAttribute(Attribute::AttrKind::Memory))
- return false;
-
- auto ME = F->getFnAttribute(Attribute::AttrKind::Memory).getMemoryEffects();
+ auto ME = Call->getMemoryEffects();
LibFunc Func;
TLI->getLibFunc(*Call, Func);
return ME.onlyWritesMemory() && ME.onlyAccessesArgPointees() &&
>From a18b4fe57f6ac415652a68c8e967d01c1be357d3 Mon Sep 17 00:00:00 2001
From: Paschalis Mpeis <paschalis.mp...@arm.com>
Date: Tue, 6 Feb 2024 09:14:50 +0000
Subject: [PATCH 5/5] Rebased and updated test after PR #80296
---
.../aarch64-veclib-function-calls-linear-ptrs.c | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
b/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
index 98085a183f46c..4a26d3ce9460d 100644
--- a/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
+++ b/clang/test/CodeGen/aarch64-veclib-function-calls-linear-ptrs.c
@@ -17,7 +17,7 @@ vectorize.
// CHECK-LABEL: define dso_local void @frexp_f64(
// CHECK-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture
noundef writeonly [[OUT1:%.*]], ptr nocapture noundef writeonly [[OUT2:%.*]],
i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK: [[CALL:%.*]] = tail call double @frexp(double noundef
[[TMP0:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR5:[0-9]+]]
+// CHECK: [[CALL:%.*]] = tail call double @frexp(double noundef
[[TMP0:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR2:[0-9]+]]
//
void frexp_f64(double *in, double *out1, int *out2, int N) {
for (int i = 0; i < N; ++i)
@@ -26,7 +26,7 @@ void frexp_f64(double *in, double *out1, int *out2, int N) {
// CHECK-LABEL: define dso_local void @frexp_f32(
// CHECK-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture
noundef writeonly [[OUT1:%.*]], ptr nocapture noundef writeonly [[OUT2:%.*]],
i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK: [[CALL:%.*]] = tail call float @frexpf(float noundef
[[TMP0:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR5]]
+// CHECK: [[CALL:%.*]] = tail call float @frexpf(float noundef
[[TMP0:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR2]]
//
void frexp_f32(float *in, float *out1, int *out2, int N) {
for (int i = 0; i < N; ++i)
@@ -35,8 +35,7 @@ void frexp_f32(float *in, float *out1, int *out2, int N) {
// CHECK-LABEL: define dso_local void @modf_f64(
// CHECK-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture
noundef writeonly [[OUT1:%.*]], ptr nocapture noundef writeonly [[OUT2:%.*]],
i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK: [[TMP11:%.*]] = tail call <vscale x 2 x double>
@armpl_svmodf_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], ptr
[[TMP10:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
-// CHECK: [[CALL:%.*]] = tail call double @modf(double noundef
[[TMP14:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR6:[0-9]+]]
+// CHECK: [[CALL:%.*]] = tail call double @modf(double noundef
[[TMP0:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR3:[0-9]+]]
//
void modf_f64(double *in, double *out1, double *out2, int N) {
for (int i = 0; i < N; ++i)
@@ -45,8 +44,7 @@ void modf_f64(double *in, double *out1, double *out2, int N) {
// CHECK-LABEL: define dso_local void @modf_f32(
// CHECK-SAME: ptr nocapture noundef readonly [[IN:%.*]], ptr nocapture
noundef writeonly [[OUT1:%.*]], ptr nocapture noundef writeonly [[OUT2:%.*]],
i32 noundef [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK: [[TMP11:%.*]] = tail call <vscale x 4 x float>
@armpl_svmodf_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], ptr
[[TMP10:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
-// CHECK: [[CALL:%.*]] = tail call float @modff(float noundef
[[TMP14:%.*]], ptr noundef [[ADD_PTR:%.*]]) #[[ATTR7:[0-9]+]]
+// CHECK: [[CALL:%.*]] = tail call float @modff(float noundef [[TMP0:%.*]],
ptr noundef [[ADD_PTR:%.*]]) #[[ATTR4:[0-9]+]]
//
void modf_f32(float *in, float *out1, float *out2, int N) {
for (int i = 0; i < N; ++i)
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
