This is an automated email from the ASF dual-hosted git repository.
tlopex pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm-tirx.git
The following commit(s) were added to refs/heads/main by this push:
new b1e553050 [FIX][CUDA] Encode the 15-bit start address of tcgen05
shared memory descriptors (#254)
b1e553050 is described below
commit b1e553050d9648f637e1ade0a1da22f2a76707d7
Author: Hongyi Jin <[email protected]>
AuthorDate: Sat Nov 21 22:58:51 2026 -0500
[FIX][CUDA] Encode the 15-bit start address of tcgen05 shared memory
descriptors (#254)
## Motivation
sm_107 lets a kernel use more than 227 KB of dynamic shared memory per
block through the oversized shared memory mode (up to 327 KB on current
parts). A main reason to use it is to keep deeper TMA pipelines or wider
tiles resident, so tcgen05 MMA operands routinely live above the 256 KB
mark.
The PTX ISA changed the tcgen05 shared memory descriptor layout to allow
that ("Shared memory descriptor" table, 9.7.18.4.1). Up to PTX ISA 9.3
the matrix start address occupied bits 0-13 with bits 14-15 reserved.
The current PTX ISA 9.5 documentation widens it to bits 0-14 (15 bits)
with only bit 15 reserved, and widens the leading dimension byte offset
to bits 16-30.
TIRx still follows the old 14-bit layout in both `SmemDescriptor` (used
by `T.cuda.tcgen05.encode_matrix_descriptor`) and the compile-time
encoder in `tile/patterns/descriptors.py`. For an operand at or above
256 KB the top address bit is dropped, so the MMA or `tcgen05.cp` reads
a different, lower shared memory region. Nothing reports an error; the
kernel just computes on the wrong data.
## Changes
- Widen `SmemDescriptor::start_address_` to 15 bits.
- Mask the shifted address with `0x7FFF` instead of `0x3FFF` in the tile
descriptor pattern, and update the encoder docstring.
Below 256 KB bit 14 is always zero, so descriptors on existing targets,
including sm_100a and sm_103a, are bit-identical. The leading dimension
byte offset is left at 14 bits; no current caller needs a larger offset.
## Testing
- The three `test_tcgen05_cp.py` source checks now expect the 15-bit
mask.
- New `test_smem_descriptor_start_address_above_256kb` (sm_107 only)
checks the runtime encoder at shared address 0x42000.
- `test_tcgen05_descriptor_encoders.py` and
`tile/cuda/test_tcgen05_cp.py`, including their GPU round trips, pass on
an sm_103a GPU.
---
python/tvm_tirx/backend/cuda/codegen/header.py | 5 +--
python/tvm_tirx/backend/cuda/cpp/descriptors.py | 2 +-
.../backend/cuda/tile/patterns/descriptors.py | 3 +-
.../codegen/test_tcgen05_descriptor_encoders.py | 40 ++++++++++++++++++++--
tests/python/tirx/tile/cuda/test_tcgen05_cp.py | 6 ++--
5 files changed, 47 insertions(+), 9 deletions(-)
diff --git a/python/tvm_tirx/backend/cuda/codegen/header.py
b/python/tvm_tirx/backend/cuda/codegen/header.py
index edb682af0..9a5114292 100644
--- a/python/tvm_tirx/backend/cuda/codegen/header.py
+++ b/python/tvm_tirx/backend/cuda/codegen/header.py
@@ -698,8 +698,9 @@ union SmemDescriptor
uint64_t desc_ = 0;
// Bitfield implementation avoids the need for shifts in assignment
struct {
- // start_address, bit [0,14), 4LSB not included
- uint16_t start_address_ : 14, : 2; // 14 bits [0,14),
2 bits unused
+ // start_address, bit [0,15), 4LSB not included. Bit 14 is only reachable
+ // with more than 256 KB of shared memory (sm_107 oversized mode).
+ uint16_t start_address_ : 15, : 1; // 15 bits [0,15),
1 bit reserved
// leading dimension byte offset, bit [16,30), 4LSB not included
uint16_t leading_byte_offset_ : 14, : 2; // 14 bits [0,14),
2 bits unused
// stride dimension byte offset, bit [32,46), 4LSB not included
diff --git a/python/tvm_tirx/backend/cuda/cpp/descriptors.py
b/python/tvm_tirx/backend/cuda/cpp/descriptors.py
index 995db8a58..3acf5ebc3 100644
--- a/python/tvm_tirx/backend/cuda/cpp/descriptors.py
+++ b/python/tvm_tirx/backend/cuda/cpp/descriptors.py
@@ -752,7 +752,7 @@ def encode_smem_descriptor_base_uint64(ldo, sdo, swizzle):
the address.
`start_address_` is the only runtime input, so the descriptor splits into a
- constant the caller bakes in and an `addr >> 4` it ORs into bits [0,14).
+ constant the caller bakes in and an `addr >> 4` it ORs into bits [0,15).
Bit layout: `SmemDescriptor` in
`python/tvm/backend/cuda/codegen/header.py`.
`ldo` / `sdo` are in 16-byte units, matching what the runtime encoder
diff --git a/python/tvm_tirx/backend/cuda/tile/patterns/descriptors.py
b/python/tvm_tirx/backend/cuda/tile/patterns/descriptors.py
index c729614db..71cda1a03 100644
--- a/python/tvm_tirx/backend/cuda/tile/patterns/descriptors.py
+++ b/python/tvm_tirx/backend/cuda/tile/patterns/descriptors.py
@@ -61,7 +61,8 @@ def _encode_matrix_descriptor(ctx, smem, ldo, sdo, swizzle,
atom_origin):
+ T.cast(swizzle_bytes(physical, swizzle), ty="uint32")
)
address = T.cast(address // T.uint32(16), ty="uint64")
- return T.bitwise_or(descriptor, T.bitwise_and(address, T.uint64(0x3FFF)))
+ # The start address field is 15 bits (PTX ISA 9.7.18.4.1, bits 0-14).
+ return T.bitwise_or(descriptor, T.bitwise_and(address, T.uint64(0x7FFF)))
def _constant_stride(ctx, difference):
diff --git a/tests/python/tirx/codegen/test_tcgen05_descriptor_encoders.py
b/tests/python/tirx/codegen/test_tcgen05_descriptor_encoders.py
index b97dd8ea7..369786fe9 100644
--- a/tests/python/tirx/codegen/test_tcgen05_descriptor_encoders.py
+++ b/tests/python/tirx/codegen/test_tcgen05_descriptor_encoders.py
@@ -57,7 +57,7 @@ def _compile_and_run(kernel, n_out):
# One case per axis the SMEM encoder branches on: every swizzle enum, and
-# offsets that exercise both 14-bit fields.
+# offsets that exercise both offset fields.
@pytest.mark.gpu
@pytest.mark.skipif(not env.has_cuda(), reason="need cuda")
@pytest.mark.parametrize(
@@ -85,13 +85,49 @@ def test_smem_descriptor_matches_runtime_encoder(ldo, sdo,
swizzle):
out[1] = T.cast(T.cuda.cvta_generic_to_shared(smem.ptr_to([0])),
ty="uint64")
got = _compile_and_run(kernel, 2)
- want = encode_smem_descriptor_base_uint64(ldo, sdo, swizzle) |
((int(got[1]) >> 4) & 0x3FFF)
+ want = encode_smem_descriptor_base_uint64(ldo, sdo, swizzle) |
((int(got[1]) >> 4) & 0x7FFF)
assert int(got[0]) == want, (
f"runtime {int(got[0]):#018x} != compile-time {want:#018x} "
f"for ldo={ldo} sdo={sdo} swizzle={swizzle}"
)
[email protected]
[email protected](
+ not env.has_cuda_compute(10, 7, exact=True), reason="need sm_107 oversized
shared memory"
+)
+def test_smem_descriptor_start_address_above_256kb():
+ """Bit 14 of the start address field is only reachable above 256 KB of
shared memory."""
+ nbytes = 300 * 1024
+ offset = 270336 # 0x42000: bit 18 of the byte address, bit 14 once encoded
+
+ @T.function
+ def kernel(out: T.Tensor((2,), "uint64")):
+ T.device_entry(
+ launch=T.cuda.LaunchConfig(
+ grid=1,
+ block=32,
+ dynamic_smem_bytes=nbytes,
+ shared_memory_mode="allow_oversized",
+ )
+ )
+ tx = T.cuda.thread_idx("x")
+ smem = T.alloc_tensor(ty_args=[T.Tensor((nbytes // 4,), "uint32",
scope="shared.dyn")])
+ smem[offset // 4 + tx] = T.uint32(0)
+ if tx == 0:
+ desc = T.local_scalar("uint64")
+ T.cuda.tcgen05.encode_matrix_descriptor(
+ T.address_of(desc), smem.ptr_to([offset // 4]), ldo=0, sdo=8,
swizzle=0
+ )
+ out[0] = desc
+ out[1] = T.cast(T.cuda.cvta_generic_to_shared(smem.ptr_to([offset
// 4])), ty="uint64")
+
+ got = _compile_and_run(kernel, 2)
+ address = int(got[1])
+ assert address >= 1 << 18
+ assert int(got[0]) & 0x7FFF == address >> 4
+
+
# (M, N, K, a_dtype, b_dtype, trans_a, trans_b, cta_group)
@pytest.mark.gpu
@pytest.mark.skipif(not env.has_cuda(), reason="need cuda")
diff --git a/tests/python/tirx/tile/cuda/test_tcgen05_cp.py
b/tests/python/tirx/tile/cuda/test_tcgen05_cp.py
index eb4fdf54d..1e4267e69 100644
--- a/tests/python/tirx/tile/cuda/test_tcgen05_cp.py
+++ b/tests/python/tirx/tile/cuda/test_tcgen05_cp.py
@@ -409,7 +409,7 @@ def test_cp_explicit_shape_and_inferred_descriptor():
ex = _compile(kernel)
assert len(_cp_call_lines(ex, "128x256b")) == 4
source = ex.mod.imports[0].inspect_source()
- assert "16383" in source # descriptor's 14-bit start-address field
+ assert "32767" in source # descriptor's 15-bit start-address field
@pytest.mark.gpu
@@ -720,7 +720,7 @@ def test_cp_explicit_32x128b_instruction_sequence():
assert len(cp_lines) == 4
# The host readback test checks the physical correspondence independently.
for line in cp_lines:
- assert "16383" in line
+ assert "32767" in line
# Invalid native opcode, ownership, alignment and descriptor contracts
@@ -1446,7 +1446,7 @@ def test_multi_cp_infers_descriptor_fields():
ex = _compile(kernel)
calls = _cp_call_lines(ex, "32x128b", "warpx4")
assert len(calls) == 4
- assert all("16383" in line for line in calls)
+ assert all("32767" in line for line in calls)
@pytest.mark.skipif(not env.has_cuda_compute(10), reason="Blackwell required")