https://github.com/ro-i updated https://github.com/llvm/llvm-project/pull/231923
>From d8d5863582454c5216a67ef38a7d0cad06503d8b Mon Sep 17 00:00:00 2001 From: Robert Imschweiler <[email protected]> Date: Wed, 14 Oct 2026 07:12:35 -0500 Subject: [PATCH 1/3] [offload] Copy the kernel launch environment with a direct-write memcpy Claude assisted with this patch. --- offload/libompaccsupport/device.cpp | 20 +++-- .../offloading/kernel_launch_environment.c | 76 +++++++++++++++++++ 2 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 offload/test/offloading/kernel_launch_environment.c diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp index e62a34a7a5991b..22919c07027cde 100644 --- a/offload/libompaccsupport/device.cpp +++ b/offload/libompaccsupport/device.cpp @@ -772,12 +772,22 @@ getKernelLaunchEnvironment(DeviceTy &Device, uint32_t OmpABIVersion, freeAfterSynchronization(Device, AsyncInfo, *ReductionAllocOrErr); } - if (Device.submitData(*AllocOrErr, &LocalKLE, - sizeof(KernelLaunchEnvironmentTy), AsyncInfo, - /*Entry=*/nullptr, /*HDTTMapPtr=*/nullptr, - /*Loc=*/nullptr, /*Name=*/"KernelLaunchEnv")) + INFO(OMP_INFOTYPE_DATA_TRANSFER, Device.DeviceID, + "Copying data from host to device, HstPtr=" DPxMOD ", TgtPtr=" DPxMOD + ", Size=%" PRId64 ", Name=KernelLaunchEnv\n", + DPxPTR(&LocalKLE), DPxPTR(*AllocOrErr), + sizeof(KernelLaunchEnvironmentTy)); + + // The allocation is fresh, so no work pending on the queue accesses it and + // the copy can bypass the queue. Without a queue, the plugin may write the + // memory directly. + if (const auto *Res = olMemcpy( + /*Queue=*/nullptr, *AllocOrErr, Device.DeviceHandle, &LocalKLE, + /*SrcDevice=*/nullptr, sizeof(KernelLaunchEnvironmentTy))) return createError(ErrorCode::BackendFailure, - "failed to copy the kernel launch environment"); + "failed to copy the kernel launch environment to " + "device %d: %s", + Device.DeviceID, Res->Details); return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr); } diff --git a/offload/test/offloading/kernel_launch_environment.c b/offload/test/offloading/kernel_launch_environment.c new file mode 100644 index 00000000000000..c6703f3b2b3805 --- /dev/null +++ b/offload/test/offloading/kernel_launch_environment.c @@ -0,0 +1,76 @@ +// Stress the kernel launch environment (KLE) of concurrent launches. +// +// A cross-team reduction gives the launch a KLE, which carries that launch's +// own reduction buffer. The KLE is copied to the device synchronously, which the +// plugin may do by storing it in place where device memory is host-writable. +// Either has to hold up when launches overlap: every launch needs its own KLE, +// and a launch's KLE has to be visible to the device before its kernel runs +// and stay intact until it finishes. Otherwise a launch reduces into another +// launch's buffer, or reads a KLE that has not landed yet. Every launch here +// accumulates a distinct value, so either shows up as a wrong sum. +// +// RUN: %libomptarget-compile-generic -fopenmp-offload-mandatory +// RUN: %libomptarget-run-generic +// RUN: %libomptarget-compileopt-generic -fopenmp-offload-mandatory +// RUN: %libomptarget-run-generic +// RUN: rm -rf %t.rr && mkdir -p %t.rr +// RUN: env LIBOMPTARGET_RECORD=1 LIBOMPTARGET_RECORD_DIR=%t.rr \ +// RUN: %libomptarget-run-generic +// +// REQUIRES: gpu + +#include <omp.h> +#include <stdio.h> + +#define NUM_LAUNCHES 32 +#define N 4096 + +// Launch k accumulates k + 1 per element. +static long expected(int k) { return (long)(k + 1) * N; } + +static int check(const char *Phase, long *Results) { + int Errors = 0; + for (int k = 0; k < NUM_LAUNCHES; k++) + if (Results[k] != expected(k)) { + fprintf(stderr, "%s: launch %d reduced to %ld, expected %ld\n", Phase, k, + Results[k], expected(k)); + Errors++; + } + return Errors; +} + +int main(void) { + static long Results[NUM_LAUNCHES]; + int Errors = 0; + + // Launches issued back to back without an intervening synchronization, so + // that several launches are in flight at once. + for (int k = 0; k < NUM_LAUNCHES; k++) { + Results[k] = 0; +#pragma omp target teams distribute parallel for map(tofrom : Results[k : 1]) \ + reduction(+ : Results[k]) firstprivate(k) nowait + for (int i = 0; i < N; i++) + Results[k] += k + 1; + } +#pragma omp taskwait + Errors += check("nowait", Results); + + // Same, but with the launches and the synchronizations spread over several + // host threads: a thread finalizing its queue must not disturb the launch + // environment of a launch another thread still has in flight. +#pragma omp parallel for num_threads(8) + for (int k = 0; k < NUM_LAUNCHES; k++) { + long Sum = 0; +#pragma omp target teams distribute parallel for map(tofrom : Sum) \ + reduction(+ : Sum) firstprivate(k) + for (int i = 0; i < N; i++) + Sum += k + 1; + Results[k] = Sum; + } + Errors += check("threaded", Results); + + if (Errors) + return 1; + printf("PASS\n"); + return 0; +} >From 614235bbfa8733171ee6ac4831db87b7db3ce070 Mon Sep 17 00:00:00 2001 From: Robert Imschweiler <[email protected]> Date: Wed, 28 Oct 2026 16:00:41 -0500 Subject: [PATCH 2/3] use submitDataSync --- offload/libompaccsupport/device.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp index 22919c07027cde..5698340083c756 100644 --- a/offload/libompaccsupport/device.cpp +++ b/offload/libompaccsupport/device.cpp @@ -781,13 +781,12 @@ getKernelLaunchEnvironment(DeviceTy &Device, uint32_t OmpABIVersion, // The allocation is fresh, so no work pending on the queue accesses it and // the copy can bypass the queue. Without a queue, the plugin may write the // memory directly. - if (const auto *Res = olMemcpy( - /*Queue=*/nullptr, *AllocOrErr, Device.DeviceHandle, &LocalKLE, - /*SrcDevice=*/nullptr, sizeof(KernelLaunchEnvironmentTy))) + if (Device.submitDataSync(*AllocOrErr, &LocalKLE, + sizeof(KernelLaunchEnvironmentTy))) return createError(ErrorCode::BackendFailure, "failed to copy the kernel launch environment to " - "device %d: %s", - Device.DeviceID, Res->Details); + "device %d", + Device.DeviceID); return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr); } >From 99f78a2b16264675358c4e03d589d1c7766f1985 Mon Sep 17 00:00:00 2001 From: Robert Imschweiler <[email protected]> Date: Thu, 29 Oct 2026 09:56:58 -0500 Subject: [PATCH 3/3] only use sync copy on AMDGPU; improve logging --- offload/libompaccsupport/device.cpp | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp index 5698340083c756..fae218efe23a59 100644 --- a/offload/libompaccsupport/device.cpp +++ b/offload/libompaccsupport/device.cpp @@ -772,17 +772,20 @@ getKernelLaunchEnvironment(DeviceTy &Device, uint32_t OmpABIVersion, freeAfterSynchronization(Device, AsyncInfo, *ReductionAllocOrErr); } - INFO(OMP_INFOTYPE_DATA_TRANSFER, Device.DeviceID, - "Copying data from host to device, HstPtr=" DPxMOD ", TgtPtr=" DPxMOD - ", Size=%" PRId64 ", Name=KernelLaunchEnv\n", - DPxPTR(&LocalKLE), DPxPTR(*AllocOrErr), - sizeof(KernelLaunchEnvironmentTy)); - // The allocation is fresh, so no work pending on the queue accesses it and - // the copy can bypass the queue. Without a queue, the plugin may write the - // memory directly. - if (Device.submitDataSync(*AllocOrErr, &LocalKLE, - sizeof(KernelLaunchEnvironmentTy))) + // the copy can bypass the queue. Without a queue, the AMDGPU plugin may write + // the memory directly. Other backends copy on the queue. + int32_t Ret = + Device.getBackend() == OL_PLATFORM_BACKEND_AMDGPU + ? Device.submitDataSync(*AllocOrErr, &LocalKLE, + sizeof(KernelLaunchEnvironmentTy), + /*Entry=*/nullptr, /*HDTTMapPtr=*/nullptr, + /*Loc=*/nullptr, /*Name=*/"KernelLaunchEnv") + : Device.submitData(*AllocOrErr, &LocalKLE, + sizeof(KernelLaunchEnvironmentTy), AsyncInfo, + /*Entry=*/nullptr, /*HDTTMapPtr=*/nullptr, + /*Loc=*/nullptr, /*Name=*/"KernelLaunchEnv"); + if (Ret) return createError(ErrorCode::BackendFailure, "failed to copy the kernel launch environment to " "device %d", _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
