This is an automated email from the ASF dual-hosted git repository.
tlopex pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm-tirx.git
The following commit(s) were added to refs/heads/main by this push:
new 5a1e069de [FIX] Keep float additions of +0.0 that change the sign of
zero (#255)
5a1e069de is described below
commit 5a1e069deb1e0444415753a83cbd71ddd9633876
Author: Hongyi Jin <[email protected]>
AuthorDate: Sat Nov 21 23:16:00 2026 -0500
[FIX] Keep float additions of +0.0 that change the sign of zero (#255)
## Motivation
`x + 0.0` is the standard way to turn -0.0 into +0.0 while leaving every
other value unchanged (IEEE 754, round to nearest: -0.0 + +0.0 = +0.0).
Kernels depend on it, for example in Lamport-style hand-offs where a
receive buffer is pre-filled with -0.0 as a "not yet written" sentinel:
the producer must canonicalize -0.0 in its payload, or the consumer
waits forever. The TMA data validity report on sm_107
(`.mbarrier::report::validity::per_16bytes::80000000`) uses the same
sentinel.
`TryConstFold<Add>` treats any float zero as an additive identity, and
this runs while TVMScript is parsed, so
```python
y[i] = x[i] * T.float32(-1.5) + T.float32(0)
```
silently becomes `y[i] = x[i] * T.float32(-1.5)`. The canonicalization
disappears and such a kernel hangs. `TryConstFold<Sub>` has the same
problem for `x - (-0.0)`.
## Changes
Under IEEE 754, `x + (-0.0)` and `x - (+0.0)` equal `x` for every `x`,
but `x + (+0.0)` and `x - (-0.0)` do not when `x` is -0.0.
- `x + (-0.0)` and `x - (+0.0)` are still folded.
- `x + (+0.0)` and `x - (-0.0)` are folded only when `x` is a constant
other than -0.0. This keeps folds such as fp16 `0 + 0.5`, which has no
direct constant evaluation path.
- The Rust port of constant folding
(`rust/tvm-tirx/src/ir/prim/const_fold.rs`) applies the same rule, so it
keeps agreeing with the C++ folder (`binary_operators_agree_with_cpp`).
- `RewriteSimplifier` dropped `x + broadcast(0.0)` for any zero, so a
vectorized `x + 0.0` lost the canonicalization too. It now folds only a
broadcast of -0.0.
`test_load_store_noop_after_simplify` relied on `A[0] + (5.0 - 5.0)`
being a no-op, which it is not for `A[0] == -0.0`. It now uses `A[0] *
(5.0 - 4.0)`, which is an identity for every value, so it still checks
no-op detection after simplification.
Other float rewrites that ignore signed zeros or NaN, such as `x * 0 ->
0` or `x - x -> 0`, are out of scope here.
## Testing
- New `test_const_fold_float_signed_zero` covers each folded and kept
form, plus the integer case.
- `test_rewrite_simplify.py`: the vector case with a `+0.0` broadcast is
now kept, and a new case checks that a `-0.0` broadcast is still folded.
- The `+ 0.0f` survives into the generated CUDA. nvcc keeps it with and
without `--use_fast_math` (as `fma.rn{.ftz}.f32 ..., 0f00000000`).
- The full non-GPU Python suite and `cargo test --manifest-path
rust/Cargo.toml` pass.
---
include/tvm/ir/prim/const_fold.h | 23 ++++++++++++++++++++---
rust/tvm-tirx/src/ir/prim/const_fold.rs | 25 ++++++++++++++++++++++---
src/sym/rewrite_simplify.cc | 5 ++++-
tests/python/sym/test_rewrite_simplify.py | 9 ++++++++-
tests/python/tirx/test_ops.py | 18 ++++++++++++++++++
tests/python/tirx/transform/test_simplify.py | 3 ++-
6 files changed, 74 insertions(+), 9 deletions(-)
diff --git a/include/tvm/ir/prim/const_fold.h b/include/tvm/ir/prim/const_fold.h
index 030652cc1..13a3807da 100644
--- a/include/tvm/ir/prim/const_fold.h
+++ b/include/tvm/ir/prim/const_fold.h
@@ -121,6 +121,19 @@ inline bool IsInFloat32Range(double value) {
value <= std::numeric_limits<float>::max());
}
+/*!
+ * \brief Whether a float constant is the additive identity for x.
+ *
+ * IEEE 754 (round to nearest): x + (-0.0) == x for every x, while x + (+0.0)
+ * maps -0.0 to +0.0. A positive zero is therefore an identity only when x is a
+ * constant that is not -0.0.
+ */
+inline bool IsFloatAddIdentity(const FloatImmObj* zero, const FloatImmObj* x) {
+ if (zero == nullptr || zero->value != 0) return false;
+ if (std::signbit(zero->value)) return true;
+ return x != nullptr && !(x->value == 0 && std::signbit(x->value));
+}
+
} // namespace detail
#define TVM_PRIM_CONST_PROPAGATION(BODY) \
@@ -161,8 +174,8 @@ inline ffi::Optional<PrimExpr>
TryConstFold<prim::Add>(PrimExpr a, PrimExpr b) {
return FloatImm(result_ty, fa->value + fb->value);
}
}
- if (fa && fa->value == 0) return b;
- if (fb && fb->value == 0) return a;
+ if (detail::IsFloatAddIdentity(fa, fb)) return b;
+ if (detail::IsFloatAddIdentity(fb, fa)) return a;
});
return std::nullopt;
}
@@ -188,7 +201,11 @@ inline ffi::Optional<PrimExpr>
TryConstFold<prim::Sub>(PrimExpr a, PrimExpr b) {
return FloatImm(result_ty, fa->value - fb->value);
}
}
- if (fb && fb->value == 0) return a;
+ // x - (+0.0) == x + (-0.0) == x for every x; x - (-0.0) == x + (+0.0).
+ if (fb && fb->value == 0 &&
+ (!std::signbit(fb->value) || (fa && !(fa->value == 0 &&
std::signbit(fa->value))))) {
+ return a;
+ }
});
return std::nullopt;
}
diff --git a/rust/tvm-tirx/src/ir/prim/const_fold.rs
b/rust/tvm-tirx/src/ir/prim/const_fold.rs
index 86049f110..4e10be16e 100644
--- a/rust/tvm-tirx/src/ir/prim/const_fold.rs
+++ b/rust/tvm-tirx/src/ir/prim/const_fold.rs
@@ -54,15 +54,30 @@ pub(super) fn add(a: &PrimExpr, b: &PrimExpr) -> Folded {
_ => {}
}
}
- if fa == Some(0.0) {
+ if is_float_add_identity(fa, fb) {
return keep(b);
}
- if fb == Some(0.0) {
+ if is_float_add_identity(fb, fa) {
return keep(a);
}
Ok(None)
}
+/// Whether the float constant `zero` is the additive identity for `x`.
+///
+/// IEEE 754 (round to nearest): `x + (-0.0) == x` for every x, while `x +
(+0.0)` maps -0.0 to
+/// +0.0. A positive zero is therefore an identity only when `x` is a constant
that is not -0.0.
+fn is_float_add_identity(zero: Option<f64>, x: Option<f64>) -> bool {
+ match zero {
+ Some(z) if z == 0.0 => z.is_sign_negative() || x.is_some_and(|x|
!is_negative_zero(x)),
+ _ => false,
+ }
+}
+
+fn is_negative_zero(x: f64) -> bool {
+ x == 0.0 && x.is_sign_negative()
+}
+
/// Fold `a - b`.
pub(super) fn sub(a: &PrimExpr, b: &PrimExpr) -> Folded {
let ty = a.prim_ty();
@@ -80,7 +95,11 @@ pub(super) fn sub(a: &PrimExpr, b: &PrimExpr) -> Folded {
_ => {}
}
}
- if fb == Some(0.0) {
+ // x - (+0.0) == x + (-0.0) == x for every x; x - (-0.0) == x + (+0.0).
+ if let Some(y) = fb
+ && y == 0.0
+ && (!y.is_sign_negative() || fa.is_some_and(|x| !is_negative_zero(x)))
+ {
return keep(a);
}
Ok(None)
diff --git a/src/sym/rewrite_simplify.cc b/src/sym/rewrite_simplify.cc
index b9511e2ba..07330b81e 100644
--- a/src/sym/rewrite_simplify.cc
+++ b/src/sym/rewrite_simplify.cc
@@ -33,6 +33,7 @@
#include <tvm/sym/analyzer.h>
#include <algorithm>
+#include <cmath>
#include <numeric>
#include <tuple>
#include <utility>
@@ -272,7 +273,9 @@ UnchangedOr<PrimExpr>
RewriteSimplifier::Impl::Mutate_(const prim::AddObj* op,
TVM_TRY_REWRITE(ramp(b1, s1, lanes) + broadcast(x, lanes), ramp(b1 + x,
s1, lanes));
TVM_TRY_REWRITE(broadcast(x, lanes) + ramp(b1, s1, lanes), ramp(x + b1,
s1, lanes));
TVM_TRY_REWRITE(broadcast(x, lanes) + broadcast(y, lanes), broadcast(x +
y, lanes));
- TVM_TRY_REWRITE_IF(x + broadcast(c4, lanes), x, c4.Eval()->value == 0.0f);
+ // Only -0.0 is an additive identity for every x: x + (+0.0) maps -0.0 to
+0.0.
+ TVM_TRY_REWRITE_IF(x + broadcast(c4, lanes), x,
+ c4.Eval()->value == 0.0 &&
std::signbit(c4.Eval()->value));
}
if (IsIndexTypedExpr(op)) {
diff --git a/tests/python/sym/test_rewrite_simplify.py
b/tests/python/sym/test_rewrite_simplify.py
index d9555ad7d..6a4616fb6 100644
--- a/tests/python/sym/test_rewrite_simplify.py
+++ b/tests/python/sym/test_rewrite_simplify.py
@@ -160,11 +160,18 @@ class TestVector(BaseCompare):
tvm_tirx.ir.prim.vector_expr.Broadcast(0, tirx.vscale() * 8) + y,
tvm_tirx.ir.prim.vector_expr.Broadcast(y, tirx.vscale() * 8),
),
+ # x + (-0.0) is an identity for every x; x + (+0.0) maps -0.0 to +0.0
and stays.
TestCase(
tvm_tirx.ir.prim.vector_expr.Ramp(x, 1, 4).astype("float32x4")
- + tvm_tirx.ir.prim.vector_expr.Broadcast(0.0, 4),
+ + tvm_tirx.ir.prim.vector_expr.Broadcast(tirx.const(-0.0,
"float32"), 4),
tvm_tirx.ir.prim.vector_expr.Ramp(x, 1, 4).astype("float32x4"),
),
+ TestCase(
+ tvm_tirx.ir.prim.vector_expr.Ramp(x, 1, 4).astype("float32x4")
+ + tvm_tirx.ir.prim.vector_expr.Broadcast(0.0, 4),
+ tvm_tirx.ir.prim.vector_expr.Ramp(x, 1, 4).astype("float32x4")
+ + tvm_tirx.ir.prim.vector_expr.Broadcast(0.0, 4),
+ ),
# Sub rules
TestCase(
tvm_tirx.ir.prim.vector_expr.Ramp(x, 4, 4) -
tvm_tirx.ir.prim.vector_expr.Ramp(y, 2, 4),
diff --git a/tests/python/tirx/test_ops.py b/tests/python/tirx/test_ops.py
index 03717478a..1c576a12f 100644
--- a/tests/python/tirx/test_ops.py
+++ b/tests/python/tirx/test_ops.py
@@ -63,6 +63,24 @@ def test_const_fold2():
assert isinstance(tdiv(1, x), tvm_tirx.tirx.Div)
+def test_const_fold_float_signed_zero():
+ """Only identities that hold for -0.0 may be folded (IEEE 754
round-to-nearest)."""
+ x = tvm_tirx.tirx.Var("x", "float32")
+ pos = tvm_tirx.tirx.const(0.0, "float32")
+ neg = tvm_tirx.tirx.const(-0.0, "float32")
+ # x + (-0.0) == x and x - (+0.0) == x for every x, including x = -0.0.
+ assert (x + neg).same_as(x)
+ assert (neg + x).same_as(x)
+ assert (x - pos).same_as(x)
+ # x + (+0.0) and x - (-0.0) map -0.0 to +0.0, so they must stay.
+ assert isinstance(x + pos, tvm_tirx.tirx.Add)
+ assert isinstance(pos + x, tvm_tirx.tirx.Add)
+ assert isinstance(x - neg, tvm_tirx.tirx.Sub)
+ # Integer zero is still an identity.
+ i = tvm_tirx.tirx.Var("i", "int32")
+ assert (i + 0).same_as(i)
+
+
def test_const_fold3():
# Test that using ints with logic operations is forbidden
x = tvm_tirx.tirx.Var("x", "int32")
diff --git a/tests/python/tirx/transform/test_simplify.py
b/tests/python/tirx/transform/test_simplify.py
index dad1daae2..a4d40a61f 100644
--- a/tests/python/tirx/transform/test_simplify.py
+++ b/tests/python/tirx/transform/test_simplify.py
@@ -155,7 +155,8 @@ def test_load_store_noop_after_simplify():
@T.function(private=True)
def before(A: T.Tensor((1,), "float32")):
- A[0] = A[0] + (5.0 - 5.0)
+ # x * 1.0 is an identity for every x; x + 0.0 is not (-0.0 + 0.0 ==
+0.0).
+ A[0] = A[0] * (5.0 - 4.0)
@T.function(private=True)
def expected(A: T.Tensor((1,), "float32")):