PR #25628 opened by Steven Xiao (younengxiao)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25628
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25628.patch
Add device=migraphx to the ONNX Runtime backend, mirroring how MIGraphX
is distributed on each platform:
- Built-in: AppendExecutionProvider("MIGraphX") via the string-keyed API
(ONNX Runtime >= 1.23), e.g. the onnxruntime-migraphx wheel on Linux.
- Plugin: new ep_library option registers a plugin EP library (ONNX
Runtime >= 1.22), such as the amdgpu-ep.dll that Windows ML installs,
since the Windows HIP SDK has no MIGraphX.
Falls back to CPU with a warning if unavailable, matching the other
execution providers.
ffmpeg -i input.mp4 -vf
"dnn_processing=dnn_backend=onnx:model=sr_model.onnx:device=migraphx" output.mp4
Tested on Windows (RX 9070 XT, plugin EP) and Linux (gfx1201, built-in
EP).
>From ad6238078579ae7cb7f3e866e48deb8e419a899b Mon Sep 17 00:00:00 2001
From: Steven Xiao <[email protected]>
Date: Fri, 30 Oct 2026 16:57:07 -0400
Subject: [PATCH] avfilter/dnn: add MIGraphX execution provider support to the
ONNX backend
Add device=migraphx to the ONNX Runtime backend, mirroring how MIGraphX
is distributed on each platform:
- Built-in: AppendExecutionProvider("MIGraphX") via the string-keyed API
(ONNX Runtime >= 1.23), e.g. the onnxruntime-migraphx wheel on Linux.
- Plugin: new ep_library option registers a plugin EP library (ONNX
Runtime >= 1.22), such as the amdgpu-ep.dll that Windows ML installs,
since the Windows HIP SDK has no MIGraphX.
Falls back to CPU with a warning if unavailable, matching the other
execution providers.
ffmpeg -i input.mp4 -vf
"dnn_processing=dnn_backend=onnx:model=sr_model.onnx:device=migraphx" output.mp4
Tested on Windows (RX 9070 XT, plugin EP) and Linux (gfx1201, built-in
EP).
Signed-off-by: younnegxiao <[email protected]>
---
doc/filters.texi | 32 +++++-
libavfilter/dnn/dnn_backend_onnx.c | 158 ++++++++++++++++++++++++++++-
libavfilter/dnn_interface.h | 1 +
3 files changed, 186 insertions(+), 5 deletions(-)
diff --git a/doc/filters.texi b/doc/filters.texi
index 37f9123df49..31e34e1a216 100644
--- a/doc/filters.texi
+++ b/doc/filters.texi
@@ -12367,6 +12367,15 @@ through the model. The output frames are D3D12
hardware frames, so they
must either be consumed by a D3D12 encoder or downloaded explicitly with
@code{hwdownload,format=nv12}.
+The @code{migraphx} device only accepts software frames: input and output
+tensors are passed to ONNX Runtime in host memory, and MIGraphX copies them
+to and from the GPU on every run. D3D12 hardware-frame input is not
+supported with @code{migraphx} and fails during configuration; use
+@option{device=dml} for the D3D12 zero-copy path. The @code{migraphx}
+device requires either ONNX Runtime 1.23 or later built with the MIGraphX
+execution provider, or a MIGraphX plugin execution provider library set
+with @option{ep_library}, such as the one Windows ML installs on Windows.
+
@end table
@item model
@@ -12412,17 +12421,24 @@ and Torch backends.
@item device
Set the device to run the model. For the ONNX Runtime backend this selects the
execution provider: @code{cpu} (default), @code{cuda} (NVIDIA GPU),
-@code{dml} (DirectML, Windows only) or @code{vitisai} (AMD Ryzen AI NPU).
+@code{dml} (DirectML, Windows only), @code{vitisai} (AMD Ryzen AI NPU) or
+@code{migraphx} (AMD GPU via MIGraphX).
@item device_id
-Set the device index used by GPU execution providers (e.g. @code{cuda} or
-@code{dml}) for the ONNX Runtime backend. Default is 0.
+Set the device index used by GPU execution providers (e.g. @code{cuda},
+@code{dml} or @code{migraphx}) for the ONNX Runtime backend. Default is 0.
@item threads_per_operation
ONNX Runtime backend only. Set the number of CPU threads used per ONNX
Runtime operator when running with @code{device=cpu}. Default is 0 (let
ONNX Runtime choose automatically). Has no effect for GPU/NPU providers.
+@item ep_library
+ONNX Runtime backend only, used with @code{device=migraphx}. Set the path of
+a plugin execution provider library to register with ONNX Runtime, for
+example the @file{amdgpu-ep.dll} that Windows ML installs for AMD GPUs.
+Requires FFmpeg to be built against ONNX Runtime 1.22 or later.
+
@end table
@subsection Examples
@@ -12482,6 +12498,16 @@ inference so that a software encoder can be used:
-y output.mp4
@end example
+@item
+Run the same model on an AMD GPU with the MIGraphX execution provider
+that Windows ML installs. The package folder name depends on the installed
+version, and the @samp{:} of the drive letter has to be escaped:
+@example
+./ffmpeg -i input.mp4 -vf
"format=grayf32,dnn_processing=dnn_backend=onnx:model=sr_model.onnx:device=migraphx\
+:ep_library='C\:/Program
Files/WindowsApps/MicrosoftCorporationII.WinML.AMD.GPU.EP.1.8_1.8.65.0_x64__8wekyb3d8bbwe/ExecutionProvider/amdgpu-ep.dll'"
\
+ -f null -
+@end example
+
@item
Process rgb24 frames with a TorchScript model using the Libtorch backend:
@example
diff --git a/libavfilter/dnn/dnn_backend_onnx.c
b/libavfilter/dnn/dnn_backend_onnx.c
index dcd843f9290..f98ad794dc5 100644
--- a/libavfilter/dnn/dnn_backend_onnx.c
+++ b/libavfilter/dnn/dnn_backend_onnx.c
@@ -54,6 +54,7 @@ typedef struct ONNXModel {
OrtSession *session;
OrtSessionOptions *session_options;
OrtAllocator *allocator;
+ char *ep_registration;
SafeQueue *request_queue;
Queue *task_queue;
Queue *lltask_queue;
@@ -87,6 +88,8 @@ typedef struct ONNXRequestItem {
static const AVOption dnn_onnx_options[] = {
{ "threads_per_operation", "number of CPU threads per ORT operator
(device=cpu only)",
OFFSET(num_threads), AV_OPT_TYPE_INT, { .i64 = 0 }, 0,
INT_MAX, FLAGS },
+ { "ep_library", "path to an ONNX Runtime plugin execution provider library
(device=migraphx only)",
+ OFFSET(ep_library), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0,
FLAGS },
{ NULL }
};
@@ -289,6 +292,15 @@ static void dnn_free_model_onnx(DNNModel **model)
g_ort->ReleaseSession(onnx_model->session);
if (onnx_model->session_options)
g_ort->ReleaseSessionOptions(onnx_model->session_options);
+#if ORT_API_VERSION >= 22
+ if (onnx_model->ep_registration) {
+ OrtStatus *status =
g_ort->UnregisterExecutionProviderLibrary(onnx_model->env,
+
onnx_model->ep_registration);
+ if (status)
+ g_ort->ReleaseStatus(status);
+ }
+#endif
+ av_freep(&onnx_model->ep_registration);
if (onnx_model->env)
g_ort->ReleaseEnv(onnx_model->env);
@@ -347,6 +359,108 @@ static void append_d3d12_dml_ep(ONNXModel *onnx_model)
}
#endif
+#if ORT_API_VERSION >= 22
+/*
+ * Registers a plugin execution provider library (e.g. the MIGraphX EP that
+ * Windows ML installs) and appends the device_id-th GPU device it exposes.
+ */
+static int append_plugin_ep(ONNXModel *onnx_model, const char *library, const
char *ep_name)
+{
+ DnnContext *ctx = onnx_model->ctx;
+ const OrtEpDevice *const *devices;
+ const OrtEpDevice **devices_before = NULL;
+ const OrtEpDevice *selected = NULL;
+ const OrtHardwareDevice *hw;
+ const char *desc;
+ size_t nb_devices, nb_devices_before;
+ int gpu_index = 0;
+ OrtStatus *status;
+#ifdef _WIN32
+ wchar_t *wlibrary = NULL;
+#endif
+
+ /* The OrtEnv is shared by the whole process, so remember which devices
+ * exist before the registration and only pick from the new ones. */
+ status = g_ort->GetEpDevices(onnx_model->env, &devices,
&nb_devices_before);
+ if (status)
+ goto fail;
+ if (nb_devices_before) {
+ devices_before = av_memdup(devices, nb_devices_before *
sizeof(*devices));
+ if (!devices_before)
+ return AVERROR(ENOMEM);
+ }
+
+ onnx_model->ep_registration = av_asprintf("%s.%p", ep_name, (void
*)onnx_model);
+ if (!onnx_model->ep_registration) {
+ av_free(devices_before);
+ return AVERROR(ENOMEM);
+ }
+#ifdef _WIN32
+ if (utf8towchar(library, &wlibrary) || !wlibrary) {
+ av_freep(&onnx_model->ep_registration);
+ av_free(devices_before);
+ return AVERROR(ENOMEM);
+ }
+ status = g_ort->RegisterExecutionProviderLibrary(onnx_model->env,
+
onnx_model->ep_registration, wlibrary);
+ av_free(wlibrary);
+#else
+ status = g_ort->RegisterExecutionProviderLibrary(onnx_model->env,
+
onnx_model->ep_registration, library);
+#endif
+ if (status) {
+ av_freep(&onnx_model->ep_registration);
+ goto fail;
+ }
+
+ status = g_ort->GetEpDevices(onnx_model->env, &devices, &nb_devices);
+ if (status)
+ goto fail;
+ for (size_t i = 0; i < nb_devices && !selected; i++) {
+ const OrtHardwareDevice *hw = g_ort->EpDevice_Device(devices[i]);
+ const char *name = g_ort->EpDevice_EpName(devices[i]);
+ int is_new = 1;
+
+ for (size_t j = 0; j < nb_devices_before; j++)
+ is_new &= devices[i] != devices_before[j];
+ av_log(ctx, AV_LOG_DEBUG, "EP device %zu: %s, hardware type %d%s\n",
i, name,
+ (int)g_ort->HardwareDevice_Type(hw), is_new ? ", added by this
registration" : "");
+ /* Some plugin EPs, such as the MIGraphX one from Windows ML, take the
+ * registration name as their EP name. */
+ if (!is_new || (strcmp(name, ep_name) && strcmp(name,
onnx_model->ep_registration)) ||
+ g_ort->HardwareDevice_Type(hw) != OrtHardwareDeviceType_GPU)
+ continue;
+ if (gpu_index++ == ctx->device_id)
+ selected = devices[i];
+ }
+ av_free(devices_before);
+ devices_before = NULL;
+ if (!selected) {
+ av_log(ctx, AV_LOG_WARNING, "'%s' provides no %s GPU device with index
%d\n",
+ library, ep_name, ctx->device_id);
+ return AVERROR(ENODEV);
+ }
+
+ status =
g_ort->SessionOptionsAppendExecutionProvider_V2(onnx_model->session_options,
+ onnx_model->env,
&selected, 1,
+ NULL, NULL, 0);
+ if (status)
+ goto fail;
+
+ hw = g_ort->EpDevice_Device(selected);
+ desc = g_ort->GetKeyValue(g_ort->HardwareDevice_Metadata(hw),
"Description");
+ av_log(ctx, AV_LOG_INFO, "Using %s from '%s' on device %d (%s)\n",
+ ep_name, library, ctx->device_id, desc ? desc : "unknown GPU");
+ return 0;
+
+fail:
+ av_free(devices_before);
+ av_log(ctx, AV_LOG_WARNING, "%s\n", g_ort->GetErrorMessage(status));
+ g_ort->ReleaseStatus(status);
+ return AVERROR_EXTERNAL;
+}
+#endif
+
static int get_input_onnx(DNNModel *model, DNNData *input, const char
*input_name)
{
ONNXModel *onnx_model = (ONNXModel *)model;
@@ -1139,6 +1253,8 @@ static DNNModel *dnn_load_model_onnx(DnnContext *ctx,
DNNFunctionType func_type,
(!ctx->device || av_strcasecmp(ctx->device, "cpu") == 0)) {
g_ort->SetIntraOpNumThreads(onnx_model->session_options,
options->num_threads);
}
+ if (options->ep_library && (!ctx->device || av_strcasecmp(ctx->device,
"migraphx") != 0))
+ av_log(ctx, AV_LOG_WARNING, "ep_library is only used with
device=migraphx, ignoring it\n");
g_ort->SetSessionGraphOptimizationLevel(onnx_model->session_options,
ORT_ENABLE_ALL);
/*
@@ -1228,14 +1344,52 @@ static DNNModel *dnn_load_model_onnx(DnnContext *ctx,
DNNFunctionType func_type,
av_log(ctx, AV_LOG_WARNING,
"VitisAI provider function not available in this ONNX
Runtime API version. Falling back to CPU.\n");
}
+ } else if (av_strcasecmp(ctx->device, "migraphx") == 0) {
+ const char* migraphx_options_keys[] = {"device_id"};
+ const char* migraphx_options_values[] = {NULL};
+ char device_id_str[32];
+ snprintf(device_id_str, sizeof(device_id_str), "%d",
ctx->device_id);
+ migraphx_options_values[0] = device_id_str;
+
+ if (options->ep_library) {
+#if ORT_API_VERSION >= 22
+ if (append_plugin_ep(onnx_model, options->ep_library,
"MIGraphXExecutionProvider") < 0)
+ av_log(ctx, AV_LOG_WARNING,
+ "Failed to enable the MIGraphX plugin execution
provider. Falling back to CPU\n");
+#else
+ av_log(ctx, AV_LOG_WARNING,
+ "ep_library requires FFmpeg to be built against ONNX
Runtime 1.22 or later. "
+ "Falling back to CPU\n");
+#endif
+ } else if (g_ort->SessionOptionsAppendExecutionProvider) {
+ /* OrtMIGraphXProviderOptions changed layout in ONNX Runtime
1.23,
+ * so use the string-keyed API, which accepts MIGraphX since
1.23. */
+ status = g_ort->SessionOptionsAppendExecutionProvider(
+ onnx_model->session_options, "MIGraphX",
+ migraphx_options_keys, migraphx_options_values, 1);
+ if (status != NULL) {
+ const char *msg = g_ort->GetErrorMessage(status);
+ av_log(ctx, AV_LOG_WARNING,
+ "Failed to enable MIGraphX (device %d): %s.
MIGraphX requires ONNX Runtime 1.23 "
+ "or later built with MIGraphX support, or a
MIGraphX plugin execution provider "
+ "set with ep_library. Falling back to CPU\n",
+ ctx->device_id, msg);
+ g_ort->ReleaseStatus(status);
+ } else {
+ av_log(ctx, AV_LOG_INFO, "Using MIGraphX execution
provider on device %d\n", ctx->device_id);
+ }
+ } else {
+ av_log(ctx, AV_LOG_WARNING,
+ "MIGraphX provider function not available in this ONNX
Runtime API version. Falling back to CPU\n");
+ }
} else {
#ifdef _WIN32
av_log(ctx, AV_LOG_WARNING,
- "Unknown device '%s'. Supported: cpu, cuda, dml, vitisai.
Using CPU\n",
+ "Unknown device '%s'. Supported: cpu, cuda, dml, vitisai,
migraphx. Using CPU\n",
ctx->device);
#else
av_log(ctx, AV_LOG_WARNING,
- "Unknown device '%s'. Supported: cpu, cuda, vitisai. Using
CPU\n",
+ "Unknown device '%s'. Supported: cpu, cuda, vitisai,
migraphx. Using CPU\n",
ctx->device);
#endif
}
diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h
index 29ba2870b26..efb9abfc691 100644
--- a/libavfilter/dnn_interface.h
+++ b/libavfilter/dnn_interface.h
@@ -142,6 +142,7 @@ typedef struct THOptions {
typedef struct ONNXOptions {
const AVClass *clazz;
int num_threads;
+ char *ep_library;
} ONNXOptions;
#endif
--
2.54.0
_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]
