================
@@ -0,0 +1,751 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
UTC_ARGS: --version 6
+; RUN: llc -march=amdgcn -mcpu=gfx900 < %s | FileCheck %s
-check-prefixes=GFX900
+; RUN: llc -march=amdgcn -mcpu=gfx942 < %s | FileCheck %s
-check-prefixes=GFX942
+; RUN: llc -march=amdgcn -mcpu=gfx1010 < %s | FileCheck %s
-check-prefixes=GFX1010
+
+; Demonstrate that wait.asyncmark is a code motion barrier for loads from LDS.
+; This is the simplest demo possible. We don't actually use async ops, but just
+; a pair of adjacent LDS loads. In the absence of the async mark, these get
+; coalesced into a wider LDS load.
+
+define void @code_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr
addrspace(3) %out) {
+; GFX900-LABEL: code_barrier:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: ds_read_b32 v0, v2
+; GFX900-NEXT: ; wait_asyncmark(0)
+; GFX900-NEXT: ds_read_b32 v1, v2 offset:4
+; GFX900-NEXT: s_waitcnt lgkmcnt(0)
+; GFX900-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX900-NEXT: ds_write_b32 v3, v0
+; GFX900-NEXT: s_waitcnt lgkmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: code_barrier:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: ds_read_b32 v0, v2
+; GFX942-NEXT: ; wait_asyncmark(0)
+; GFX942-NEXT: ds_read_b32 v1, v2 offset:4
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX942-NEXT: ds_write_b32 v3, v0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1010-LABEL: code_barrier:
+; GFX1010: ; %bb.0:
+; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT: ds_read_b32 v0, v2
+; GFX1010-NEXT: ; wait_asyncmark(0)
+; GFX1010-NEXT: ds_read_b32 v1, v2 offset:4
+; GFX1010-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1010-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX1010-NEXT: ds_write_b32 v3, v0
+; GFX1010-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1010-NEXT: s_setpc_b64 s[30:31]
+ %lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
+ %val1 = load i32, ptr addrspace(3) %lds
+ call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ %val2 = load i32, ptr addrspace(3) %lds_gep1
+ %sum = add i32 %val1, %val2
+ store i32 %sum, ptr addrspace(3) %out
+ ret void
+}
+
+; Test async mark/wait with global_load_lds and global loads
+; This version uses wave barriers to enforce program order so that unrelated
vmem
+; instructions do not get reordered before reaching this point.
+
+define void @interleaved_global_and_dma(ptr addrspace(1) %foo, ptr
addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out) {
+; GFX900-LABEL: interleaved_global_and_dma:
+; GFX900: ; %bb.0: ; %entry
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_readfirstlane_b32 s4, v2
+; GFX900-NEXT: global_load_dword v7, v[3:4], off
+; GFX900-NEXT: global_load_dword v8, v[0:1], off
+; GFX900-NEXT: s_mov_b32 m0, s4
+; GFX900-NEXT: ; wave barrier
+; GFX900-NEXT: s_nop 0
+; GFX900-NEXT: global_load_dword v[3:4], off lds
+; GFX900-NEXT: ; asyncmark
+; GFX900-NEXT: global_load_dword v0, v[0:1], off
+; GFX900-NEXT: ; wave barrier
+; GFX900-NEXT: s_nop 0
+; GFX900-NEXT: global_load_dword v[3:4], off lds
+; GFX900-NEXT: ; wave barrier
+; GFX900-NEXT: global_load_dword v1, v[3:4], off
+; GFX900-NEXT: ; asyncmark
+; GFX900-NEXT: ; wait_asyncmark(1)
+; GFX900-NEXT: s_waitcnt vmcnt(3)
+; GFX900-NEXT: ds_read_b32 v3, v2
+; GFX900-NEXT: ; wait_asyncmark(0)
+; GFX900-NEXT: s_waitcnt vmcnt(1)
+; GFX900-NEXT: ds_read_b32 v2, v2
+; GFX900-NEXT: v_add_u32_e32 v4, v8, v7
+; GFX900-NEXT: s_waitcnt lgkmcnt(1)
+; GFX900-NEXT: v_add3_u32 v0, v4, v3, v0
+; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_add3_u32 v0, v0, v1, v2
+; GFX900-NEXT: global_store_dword v[5:6], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: interleaved_global_and_dma:
+; GFX942: ; %bb.0: ; %entry
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v9, v4
+; GFX942-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-NEXT: v_readfirstlane_b32 s0, v2
+; GFX942-NEXT: global_load_dword v3, v[8:9], off
+; GFX942-NEXT: global_load_dword v4, v[0:1], off
+; GFX942-NEXT: s_mov_b32 m0, s0
+; GFX942-NEXT: ; wave barrier
+; GFX942-NEXT: v_mov_b32_e32 v7, v6
+; GFX942-NEXT: global_load_lds_dword v[8:9], off
+; GFX942-NEXT: ; asyncmark
+; GFX942-NEXT: global_load_dword v0, v[0:1], off
+; GFX942-NEXT: ; wave barrier
+; GFX942-NEXT: v_mov_b32_e32 v6, v5
+; GFX942-NEXT: global_load_lds_dword v[8:9], off
+; GFX942-NEXT: ; wave barrier
+; GFX942-NEXT: global_load_dword v1, v[8:9], off
+; GFX942-NEXT: ; asyncmark
+; GFX942-NEXT: ; wait_asyncmark(1)
+; GFX942-NEXT: s_waitcnt vmcnt(3)
+; GFX942-NEXT: ds_read_b32 v5, v2
+; GFX942-NEXT: ; wait_asyncmark(0)
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: ds_read_b32 v2, v2
+; GFX942-NEXT: v_add_u32_e32 v3, v4, v3
+; GFX942-NEXT: s_waitcnt lgkmcnt(1)
+; GFX942-NEXT: v_add3_u32 v0, v3, v5, v0
+; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_add3_u32 v0, v0, v1, v2
+; GFX942-NEXT: global_store_dword v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1010-LABEL: interleaved_global_and_dma:
+; GFX1010: ; %bb.0: ; %entry
+; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT: v_readfirstlane_b32 s4, v2
+; GFX1010-NEXT: global_load_dword v7, v[3:4], off
+; GFX1010-NEXT: global_load_dword v8, v[0:1], off
+; GFX1010-NEXT: ; wave barrier
+; GFX1010-NEXT: s_mov_b32 m0, s4
+; GFX1010-NEXT: global_load_dword v[3:4], off lds
+; GFX1010-NEXT: ; asyncmark
+; GFX1010-NEXT: global_load_dword v0, v[0:1], off
+; GFX1010-NEXT: ; wave barrier
+; GFX1010-NEXT: global_load_dword v[3:4], off lds
+; GFX1010-NEXT: ; wave barrier
+; GFX1010-NEXT: global_load_dword v1, v[3:4], off
+; GFX1010-NEXT: ; asyncmark
+; GFX1010-NEXT: ; wait_asyncmark(1)
+; GFX1010-NEXT: s_waitcnt vmcnt(3)
+; GFX1010-NEXT: ds_read_b32 v3, v2
+; GFX1010-NEXT: ; wait_asyncmark(0)
+; GFX1010-NEXT: s_waitcnt vmcnt(1)
+; GFX1010-NEXT: ds_read_b32 v2, v2
+; GFX1010-NEXT: v_add_nc_u32_e32 v4, v8, v7
+; GFX1010-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1010-NEXT: v_add3_u32 v0, v4, v3, v0
+; GFX1010-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1010-NEXT: v_add3_u32 v0, v0, v1, v2
+; GFX1010-NEXT: global_store_dword v[5:6], v0, off
+; GFX1010-NEXT: s_setpc_b64 s[30:31]
+entry:
+ ; First batch: global load, global load, async global-to-LDS
+ %bar_v11 = load i32, ptr addrspace(1) %bar
+ %foo_v1 = load i32, ptr addrspace(1) %foo
+ call void @llvm.amdgcn.wave.barrier()
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr
addrspace(3) %lds, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+
+ ; Second batch: global load, async global-to-LDS, global load
+ %foo_v2 = load i32, ptr addrspace(1) %foo
+ call void @llvm.amdgcn.wave.barrier()
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr
addrspace(3) %lds, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.wave.barrier()
+ %bar_v12 = load i32, ptr addrspace(1) %bar
+ call void @llvm.amdgcn.asyncmark()
+
+ ; Wait for first async mark and read from LDS
+ ; This results in vmcnt(3) corresponding to the second batch.
+ call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ %lds_val21 = load i32, ptr addrspace(3) %lds
+
+ ; Wait for the next lds dma
+ ; This results in vmcnt(1), corresponding to %bar_v12. Could have been
combined with the lgkmcnt(1) for %lds_val21.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ %lds_val22 = load i32, ptr addrspace(3) %lds
+ %sum1 = add i32 %foo_v1, %bar_v11
+ %sum2 = add i32 %sum1, %lds_val21
+ %sum3 = add i32 %sum2, %foo_v2
+ ; Finally a vmcnt(0) for %bar_v12, which was not included in the async mark
that followed it.
+ %sum4 = add i32 %sum3, %bar_v12
+ %sum5 = add i32 %sum4, %lds_val22
+ store i32 %sum5, ptr addrspace(1) %out
+
+ ret void
+}
+
+define void @interleaved_buffer_and_dma(ptr addrspace(8) inreg %buf, ptr
addrspace(1) %foo, ptr addrspace(3) inreg %lds, ptr addrspace(1) %bar, ptr
addrspace(1) %out) {
+; GFX900-LABEL: interleaved_buffer_and_dma:
+; GFX900: ; %bb.0: ; %entry
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: s_mov_b32 m0, s20
+; GFX900-NEXT: global_load_dword v6, v[2:3], off
+; GFX900-NEXT: global_load_dword v7, v[0:1], off
+; GFX900-NEXT: v_mov_b32_e32 v8, 0x54
+; GFX900-NEXT: ; wave barrier
+; GFX900-NEXT: buffer_load_dword v8, s[16:19], 0 offen lds
+; GFX900-NEXT: ; asyncmark
+; GFX900-NEXT: global_load_dword v0, v[0:1], off
+; GFX900-NEXT: v_mov_b32_e32 v1, 0x58
+; GFX900-NEXT: ; wave barrier
+; GFX900-NEXT: buffer_load_dword v1, s[16:19], 0 offen lds
+; GFX900-NEXT: ; wave barrier
+; GFX900-NEXT: global_load_dword v1, v[2:3], off
+; GFX900-NEXT: v_mov_b32_e32 v2, s20
+; GFX900-NEXT: ; asyncmark
+; GFX900-NEXT: ; wait_asyncmark(1)
+; GFX900-NEXT: s_waitcnt vmcnt(3)
+; GFX900-NEXT: ds_read_b32 v3, v2
+; GFX900-NEXT: ; wait_asyncmark(0)
+; GFX900-NEXT: s_waitcnt vmcnt(1)
+; GFX900-NEXT: ds_read_b32 v2, v2
+; GFX900-NEXT: v_add_u32_e32 v6, v7, v6
+; GFX900-NEXT: s_waitcnt lgkmcnt(1)
+; GFX900-NEXT: v_add3_u32 v0, v6, v3, v0
+; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_add3_u32 v0, v0, v1, v2
+; GFX900-NEXT: global_store_dword v[4:5], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: interleaved_buffer_and_dma:
+; GFX942: ; %bb.0: ; %entry
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 m0, s16
+; GFX942-NEXT: global_load_dword v6, v[2:3], off
+; GFX942-NEXT: global_load_dword v7, v[0:1], off
+; GFX942-NEXT: v_mov_b32_e32 v8, 0x54
+; GFX942-NEXT: ; wave barrier
+; GFX942-NEXT: buffer_load_dword v8, s[0:3], 0 offen lds
+; GFX942-NEXT: ; asyncmark
+; GFX942-NEXT: global_load_dword v0, v[0:1], off
+; GFX942-NEXT: v_mov_b32_e32 v1, 0x58
+; GFX942-NEXT: ; wave barrier
+; GFX942-NEXT: buffer_load_dword v1, s[0:3], 0 offen lds
+; GFX942-NEXT: ; wave barrier
+; GFX942-NEXT: global_load_dword v1, v[2:3], off
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: ; asyncmark
+; GFX942-NEXT: ; wait_asyncmark(1)
+; GFX942-NEXT: s_waitcnt vmcnt(3)
+; GFX942-NEXT: ds_read_b32 v3, v2
+; GFX942-NEXT: ; wait_asyncmark(0)
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: ds_read_b32 v2, v2
+; GFX942-NEXT: v_add_u32_e32 v6, v7, v6
+; GFX942-NEXT: s_waitcnt lgkmcnt(1)
+; GFX942-NEXT: v_add3_u32 v0, v6, v3, v0
+; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_add3_u32 v0, v0, v1, v2
+; GFX942-NEXT: global_store_dword v[4:5], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1010-LABEL: interleaved_buffer_and_dma:
+; GFX1010: ; %bb.0: ; %entry
+; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT: v_mov_b32_e32 v6, 0x54
+; GFX1010-NEXT: global_load_dword v7, v[2:3], off
+; GFX1010-NEXT: global_load_dword v8, v[0:1], off
+; GFX1010-NEXT: s_mov_b32 m0, s20
+; GFX1010-NEXT: ; wave barrier
+; GFX1010-NEXT: buffer_load_dword v6, s[16:19], 0 offen lds
+; GFX1010-NEXT: ; asyncmark
+; GFX1010-NEXT: v_mov_b32_e32 v6, 0x58
+; GFX1010-NEXT: global_load_dword v0, v[0:1], off
+; GFX1010-NEXT: ; wave barrier
+; GFX1010-NEXT: buffer_load_dword v6, s[16:19], 0 offen lds
+; GFX1010-NEXT: ; wave barrier
+; GFX1010-NEXT: global_load_dword v1, v[2:3], off
+; GFX1010-NEXT: v_mov_b32_e32 v2, s20
+; GFX1010-NEXT: ; asyncmark
+; GFX1010-NEXT: ; wait_asyncmark(1)
+; GFX1010-NEXT: s_waitcnt vmcnt(3)
+; GFX1010-NEXT: ds_read_b32 v3, v2
+; GFX1010-NEXT: ; wait_asyncmark(0)
+; GFX1010-NEXT: s_waitcnt vmcnt(1)
+; GFX1010-NEXT: ds_read_b32 v2, v2
+; GFX1010-NEXT: v_add_nc_u32_e32 v6, v8, v7
+; GFX1010-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1010-NEXT: v_add3_u32 v0, v6, v3, v0
+; GFX1010-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1010-NEXT: v_add3_u32 v0, v0, v1, v2
+; GFX1010-NEXT: global_store_dword v[4:5], v0, off
+; GFX1010-NEXT: s_setpc_b64 s[30:31]
+entry:
+ ; First batch: global load, global load, async global-to-LDS.
+ %bar_v11 = load i32, ptr addrspace(1) %bar
+ %foo_v1 = load i32, ptr addrspace(1) %foo
+ call void @llvm.amdgcn.wave.barrier()
+ call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf,
ptr addrspace(3) %lds, i32 4, i32 84, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+
+ ; Second batch: global load, async global-to-LDS, global load.
+ %foo_v2 = load i32, ptr addrspace(1) %foo
+ call void @llvm.amdgcn.wave.barrier()
+ call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %buf,
ptr addrspace(3) %lds, i32 4, i32 88, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.wave.barrier()
+ %bar_v12 = load i32, ptr addrspace(1) %bar
+ call void @llvm.amdgcn.asyncmark()
+
+ ; Wait for first async mark and read from LDS.
+ ; This results in vmcnt(3) corresponding to the second batch.
+ call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ %lds_val21 = load i32, ptr addrspace(3) %lds
+
+ ; Wait for the next lds dma.
+ ; This results in vmcnt(1) because the last global load is not async.
+ call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ %lds_val22 = load i32, ptr addrspace(3) %lds
+ %sum1 = add i32 %foo_v1, %bar_v11
+ %sum2 = add i32 %sum1, %lds_val21
+ %sum3 = add i32 %sum2, %foo_v2
+ %sum4 = add i32 %sum3, %bar_v12
+ %sum5 = add i32 %sum4, %lds_val22
+ store i32 %sum5, ptr addrspace(1) %out
+
+ ret void
+}
+
+; A perfect loop that is unlikely to exist in real life. It uses only async LDS
+; DMA operations, and result in vmcnt waits that exactly match the stream of
+; those outstanding operations.
+
+define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds,
ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
+; GFX900-LABEL: test_pipelined_loop:
+; GFX900: ; %bb.0: ; %prolog
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_readfirstlane_b32 s4, v2
+; GFX900-NEXT: s_mov_b32 m0, s4
+; GFX900-NEXT: v_mov_b32_e32 v5, 0
+; GFX900-NEXT: global_load_dword v[0:1], off lds
+; GFX900-NEXT: ; asyncmark
+; GFX900-NEXT: global_load_dword v[0:1], off lds
+; GFX900-NEXT: s_mov_b32 s6, 2
+; GFX900-NEXT: s_mov_b64 s[4:5], 0
+; GFX900-NEXT: ; asyncmark
+; GFX900-NEXT: .LBB3_1: ; %loop_body
+; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX900-NEXT: v_readfirstlane_b32 s7, v2
+; GFX900-NEXT: s_mov_b32 m0, s7
+; GFX900-NEXT: s_add_i32 s6, s6, 1
+; GFX900-NEXT: global_load_dword v[0:1], off lds
+; GFX900-NEXT: ; asyncmark
+; GFX900-NEXT: ; wait_asyncmark(2)
+; GFX900-NEXT: s_waitcnt vmcnt(2)
+; GFX900-NEXT: ds_read_b32 v6, v2
+; GFX900-NEXT: v_cmp_ge_i32_e32 vcc, s6, v7
+; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX900-NEXT: s_waitcnt lgkmcnt(0)
+; GFX900-NEXT: v_add_u32_e32 v5, v5, v6
+; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX900-NEXT: s_cbranch_execnz .LBB3_1
+; GFX900-NEXT: ; %bb.2: ; %epilog
+; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX900-NEXT: ; wait_asyncmark(1)
+; GFX900-NEXT: s_waitcnt vmcnt(1)
+; GFX900-NEXT: ds_read_b32 v0, v2
+; GFX900-NEXT: ; wait_asyncmark(0)
+; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_add_u32_e32 v0, v5, v0
+; GFX900-NEXT: global_store_dword v[3:4], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: test_pipelined_loop:
+; GFX942: ; %bb.0: ; %prolog
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_readfirstlane_b32 s0, v2
+; GFX942-NEXT: s_mov_b32 m0, s0
+; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: global_load_lds_dword v[0:1], off
+; GFX942-NEXT: ; asyncmark
+; GFX942-NEXT: global_load_lds_dword v[0:1], off
+; GFX942-NEXT: v_mov_b32_e32 v4, v3
+; GFX942-NEXT: s_mov_b32 s2, 2
+; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_mov_b32_e32 v3, 0
+; GFX942-NEXT: ; asyncmark
+; GFX942-NEXT: .LBB3_1: ; %loop_body
+; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: v_readfirstlane_b32 s3, v2
+; GFX942-NEXT: s_mov_b32 m0, s3
+; GFX942-NEXT: s_add_i32 s2, s2, 1
+; GFX942-NEXT: global_load_lds_dword v[0:1], off
+; GFX942-NEXT: ; asyncmark
+; GFX942-NEXT: ; wait_asyncmark(2)
+; GFX942-NEXT: s_waitcnt vmcnt(2)
+; GFX942-NEXT: ds_read_b32 v6, v2
+; GFX942-NEXT: v_cmp_ge_i32_e32 vcc, s2, v7
+; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_add_u32_e32 v3, v3, v6
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB3_1
+; GFX942-NEXT: ; %bb.2: ; %epilog
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: ; wait_asyncmark(1)
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: ds_read_b32 v0, v2
+; GFX942-NEXT: ; wait_asyncmark(0)
+; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_add_u32_e32 v0, v3, v0
+; GFX942-NEXT: global_store_dword v[4:5], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1010-LABEL: test_pipelined_loop:
+; GFX1010: ; %bb.0: ; %prolog
+; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT: v_readfirstlane_b32 s4, v2
+; GFX1010-NEXT: v_mov_b32_e32 v5, 0
+; GFX1010-NEXT: s_mov_b32 s5, 2
+; GFX1010-NEXT: s_mov_b32 m0, s4
+; GFX1010-NEXT: s_mov_b32 s4, 0
+; GFX1010-NEXT: global_load_dword v[0:1], off lds
+; GFX1010-NEXT: ; asyncmark
+; GFX1010-NEXT: global_load_dword v[0:1], off lds
+; GFX1010-NEXT: ; asyncmark
+; GFX1010-NEXT: .LBB3_1: ; %loop_body
+; GFX1010-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1010-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010-NEXT: s_add_i32 s5, s5, 1
+; GFX1010-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v7
+; GFX1010-NEXT: s_mov_b32 m0, s6
+; GFX1010-NEXT: global_load_dword v[0:1], off lds
+; GFX1010-NEXT: ; asyncmark
+; GFX1010-NEXT: ; wait_asyncmark(2)
+; GFX1010-NEXT: s_waitcnt vmcnt(2)
+; GFX1010-NEXT: ds_read_b32 v6, v2
+; GFX1010-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1010-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1010-NEXT: v_add_nc_u32_e32 v5, v5, v6
+; GFX1010-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT: s_cbranch_execnz .LBB3_1
+; GFX1010-NEXT: ; %bb.2: ; %epilog
+; GFX1010-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT: ; wait_asyncmark(1)
+; GFX1010-NEXT: s_waitcnt vmcnt(1)
+; GFX1010-NEXT: ds_read_b32 v0, v2
+; GFX1010-NEXT: ; wait_asyncmark(0)
+; GFX1010-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1010-NEXT: v_add_nc_u32_e32 v0, v5, v0
+; GFX1010-NEXT: global_store_dword v[3:4], v0, off
+; GFX1010-NEXT: s_setpc_b64 s[30:31]
+prolog:
+ ; Load first iteration
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr
addrspace(3) %lds, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+
+ ; Load second iteration
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr
addrspace(3) %lds, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+
+ br label %loop_body
+
+loop_body:
+ %i = phi i32 [ 2, %prolog ], [ %i.next, %loop_body ]
+ %sum = phi i32 [ 0, %prolog ], [ %sum_i, %loop_body ]
+
+ ; Load next iteration
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr
addrspace(3) %lds, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+
+ ; Wait for iteration i-2 and process
+ call void @llvm.amdgcn.wait.asyncmark(i16 2)
+ %lds_idx = sub i32 %i, 2
+ %lds_val = load i32, ptr addrspace(3) %lds
+
+ %sum_i = add i32 %sum, %lds_val
+
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop_body, label %epilog
+
+epilog:
+ ; Process remaining iterations
+ call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ %lds_val_n_2 = load i32, ptr addrspace(3) %lds
+ %sum_e2 = add i32 %sum_i, %lds_val_n_2
+
+ call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ %lds_val_n_1 = load i32, ptr addrspace(3) %lds
+ %sum_e1 = add i32 %sum_e2, %lds_val_n_1
+ store i32 %sum_e2, ptr addrspace(1) %bar
+
+ ret void
+}
+
+; Software pipelined loop with async global-to-LDS and global loads
+
+define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr
addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
+; GFX900-LABEL: test_pipelined_loop_with_global:
+; GFX900: ; %bb.0: ; %prolog
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_readfirstlane_b32 s4, v2
+; GFX900-NEXT: s_mov_b32 m0, s4
+; GFX900-NEXT: global_load_dword v10, v[0:1], off
+; GFX900-NEXT: global_load_dword v14, v[3:4], off
+; GFX900-NEXT: s_mov_b32 s6, 2
+; GFX900-NEXT: global_load_dword v[0:1], off lds
+; GFX900-NEXT: ; asyncmark
+; GFX900-NEXT: global_load_dword v8, v[0:1], off
+; GFX900-NEXT: global_load_dword v9, v[3:4], off
+; GFX900-NEXT: s_mov_b64 s[4:5], 0
+; GFX900-NEXT: global_load_dword v[0:1], off lds
+; GFX900-NEXT: ; asyncmark
+; GFX900-NEXT: s_waitcnt vmcnt(2)
+; GFX900-NEXT: v_mov_b32_e32 v13, v8
+; GFX900-NEXT: s_waitcnt vmcnt(1)
+; GFX900-NEXT: v_mov_b32_e32 v15, v9
+; GFX900-NEXT: .LBB4_1: ; %loop_body
+; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX900-NEXT: v_readfirstlane_b32 s7, v2
+; GFX900-NEXT: s_waitcnt vmcnt(1)
+; GFX900-NEXT: v_mov_b32_e32 v12, v15
+; GFX900-NEXT: v_mov_b32_e32 v11, v13
+; GFX900-NEXT: global_load_dword v13, v[0:1], off
+; GFX900-NEXT: global_load_dword v15, v[3:4], off
+; GFX900-NEXT: s_mov_b32 m0, s7
+; GFX900-NEXT: s_add_i32 s6, s6, 1
+; GFX900-NEXT: global_load_dword v[0:1], off lds
+; GFX900-NEXT: v_cmp_ge_i32_e32 vcc, s6, v7
+; GFX900-NEXT: v_mov_b32_e32 v16, v14
+; GFX900-NEXT: v_mov_b32_e32 v17, v10
+; GFX900-NEXT: v_mov_b32_e32 v10, v8
+; GFX900-NEXT: v_mov_b32_e32 v14, v9
+; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX900-NEXT: ; asyncmark
+; GFX900-NEXT: ; wait_asyncmark(2)
+; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX900-NEXT: s_cbranch_execnz .LBB4_1
+; GFX900-NEXT: ; %bb.2: ; %epilog
+; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX900-NEXT: ds_read_b32 v0, v2
+; GFX900-NEXT: ; wait_asyncmark(1)
+; GFX900-NEXT: s_waitcnt vmcnt(3)
+; GFX900-NEXT: ds_read_b32 v1, v2
+; GFX900-NEXT: ; wait_asyncmark(0)
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ds_read_b32 v2, v2
+; GFX900-NEXT: v_add_u32_e32 v3, v17, v16
+; GFX900-NEXT: s_waitcnt lgkmcnt(2)
+; GFX900-NEXT: v_add3_u32 v0, v3, v0, v12
+; GFX900-NEXT: s_waitcnt lgkmcnt(1)
+; GFX900-NEXT: v_add3_u32 v0, v11, v0, v1
+; GFX900-NEXT: v_add_u32_e32 v1, v13, v15
+; GFX900-NEXT: s_waitcnt lgkmcnt(0)
+; GFX900-NEXT: v_add3_u32 v0, v1, v2, v0
+; GFX900-NEXT: global_store_dword v[5:6], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: test_pipelined_loop_with_global:
+; GFX942: ; %bb.0: ; %prolog
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_readfirstlane_b32 s0, v2
+; GFX942-NEXT: s_mov_b32 m0, s0
+; GFX942-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-NEXT: v_mov_b32_e32 v10, v3
+; GFX942-NEXT: global_load_dword v16, v[0:1], off
+; GFX942-NEXT: global_load_dword v17, v[10:11], off
+; GFX942-NEXT: v_mov_b32_e32 v9, v6
+; GFX942-NEXT: global_load_lds_dword v[0:1], off
+; GFX942-NEXT: ; asyncmark
+; GFX942-NEXT: global_load_dword v14, v[0:1], off
+; GFX942-NEXT: global_load_dword v15, v[10:11], off
+; GFX942-NEXT: v_mov_b32_e32 v8, v5
+; GFX942-NEXT: global_load_lds_dword v[0:1], off
+; GFX942-NEXT: s_mov_b32 s2, 2
+; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: ; asyncmark
+; GFX942-NEXT: s_waitcnt vmcnt(2)
+; GFX942-NEXT: v_mov_b32_e32 v18, v14
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_mov_b32_e32 v19, v15
+; GFX942-NEXT: .LBB4_1: ; %loop_body
+; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: global_load_dword v4, v[10:11], off
+; GFX942-NEXT: v_readfirstlane_b32 s3, v2
+; GFX942-NEXT: s_mov_b32 m0, s3
+; GFX942-NEXT: s_add_i32 s2, s2, 1
+; GFX942-NEXT: global_load_lds_dword v[0:1], off
+; GFX942-NEXT: v_cmp_ge_i32_e32 vcc, s2, v7
+; GFX942-NEXT: v_mov_b32_e32 v5, v16
+; GFX942-NEXT: v_mov_b32_e32 v12, v17
+; GFX942-NEXT: v_mov_b32_e32 v6, v18
+; GFX942-NEXT: v_mov_b32_e32 v13, v19
+; GFX942-NEXT: v_mov_b32_e32 v16, v14
+; GFX942-NEXT: v_mov_b32_e32 v17, v15
+; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: ; asyncmark
+; GFX942-NEXT: ; wait_asyncmark(2)
+; GFX942-NEXT: s_waitcnt vmcnt(2)
+; GFX942-NEXT: v_mov_b32_e32 v18, v3
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_mov_b32_e32 v19, v4
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB4_1
+; GFX942-NEXT: ; %bb.2: ; %epilog
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: ds_read_b32 v0, v2
+; GFX942-NEXT: ; wait_asyncmark(1)
+; GFX942-NEXT: ds_read_b32 v1, v2
+; GFX942-NEXT: ; wait_asyncmark(0)
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: ds_read_b32 v2, v2
+; GFX942-NEXT: v_add_u32_e32 v5, v5, v12
+; GFX942-NEXT: s_waitcnt lgkmcnt(2)
+; GFX942-NEXT: v_add3_u32 v0, v5, v0, v13
+; GFX942-NEXT: s_waitcnt lgkmcnt(1)
+; GFX942-NEXT: v_add3_u32 v0, v6, v0, v1
+; GFX942-NEXT: v_add_u32_e32 v1, v3, v4
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_add3_u32 v0, v1, v2, v0
+; GFX942-NEXT: global_store_dword v[8:9], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1010-LABEL: test_pipelined_loop_with_global:
+; GFX1010: ; %bb.0: ; %prolog
+; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT: v_readfirstlane_b32 s4, v2
+; GFX1010-NEXT: global_load_dword v9, v[0:1], off
+; GFX1010-NEXT: global_load_dword v13, v[3:4], off
+; GFX1010-NEXT: s_mov_b32 s5, 2
+; GFX1010-NEXT: s_mov_b32 m0, s4
+; GFX1010-NEXT: s_mov_b32 s4, 0
+; GFX1010-NEXT: global_load_dword v[0:1], off lds
+; GFX1010-NEXT: ; asyncmark
+; GFX1010-NEXT: global_load_dword v10, v[0:1], off
+; GFX1010-NEXT: global_load_dword v12, v[3:4], off
+; GFX1010-NEXT: global_load_dword v[0:1], off lds
+; GFX1010-NEXT: ; asyncmark
+; GFX1010-NEXT: s_waitcnt vmcnt(2)
+; GFX1010-NEXT: v_mov_b32_e32 v8, v10
+; GFX1010-NEXT: s_waitcnt vmcnt(1)
+; GFX1010-NEXT: v_mov_b32_e32 v11, v12
+; GFX1010-NEXT: .LBB4_1: ; %loop_body
+; GFX1010-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1010-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010-NEXT: s_waitcnt vmcnt(1)
+; GFX1010-NEXT: v_mov_b32_e32 v15, v11
+; GFX1010-NEXT: v_mov_b32_e32 v14, v8
+; GFX1010-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1010-NEXT: global_load_dword v8, v[0:1], off
+; GFX1010-NEXT: global_load_dword v11, v[3:4], off
+; GFX1010-NEXT: s_add_i32 s5, s5, 1
+; GFX1010-NEXT: s_mov_b32 m0, s6
+; GFX1010-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v7
+; GFX1010-NEXT: global_load_dword v[0:1], off lds
+; GFX1010-NEXT: v_mov_b32_e32 v16, v13
+; GFX1010-NEXT: v_mov_b32_e32 v17, v9
+; GFX1010-NEXT: v_mov_b32_e32 v9, v10
+; GFX1010-NEXT: v_mov_b32_e32 v13, v12
+; GFX1010-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1010-NEXT: ; asyncmark
+; GFX1010-NEXT: ; wait_asyncmark(2)
+; GFX1010-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT: s_cbranch_execnz .LBB4_1
+; GFX1010-NEXT: ; %bb.2: ; %epilog
+; GFX1010-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT: ds_read_b32 v0, v2
+; GFX1010-NEXT: ; wait_asyncmark(1)
+; GFX1010-NEXT: s_waitcnt vmcnt(3)
+; GFX1010-NEXT: ds_read_b32 v1, v2
+; GFX1010-NEXT: ; wait_asyncmark(0)
+; GFX1010-NEXT: s_waitcnt vmcnt(0)
+; GFX1010-NEXT: ds_read_b32 v2, v2
+; GFX1010-NEXT: v_add_nc_u32_e32 v3, v17, v16
+; GFX1010-NEXT: s_waitcnt lgkmcnt(2)
+; GFX1010-NEXT: v_add3_u32 v0, v3, v0, v15
+; GFX1010-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1010-NEXT: v_add3_u32 v0, v14, v0, v1
+; GFX1010-NEXT: v_add_nc_u32_e32 v1, v8, v11
+; GFX1010-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1010-NEXT: v_add3_u32 v0, v1, v2, v0
+; GFX1010-NEXT: global_store_dword v[5:6], v0, off
+; GFX1010-NEXT: s_setpc_b64 s[30:31]
+prolog:
+ ; Load first iteration
+ %v0 = load i32, ptr addrspace(1) %foo
+ %g0 = load i32, ptr addrspace(1) %bar
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr
addrspace(3) %lds, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+
+ ; Load second iteration
+ %v1 = load i32, ptr addrspace(1) %foo
+ %g1 = load i32, ptr addrspace(1) %bar
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %foo, ptr
addrspace(3) %lds, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+
+ br label %loop_body
+
+ ; The vmcnt at the end of the prolog and at the start of the loop header
seems
----------------
ssahasra wrote:
This is an important observation that might have a moderating effect on the
practical gains in a "real" application.
https://github.com/llvm/llvm-project/pull/173259
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
