PR #24353 opened by dongnguyenminhanh URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24353 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24353.patch
Add dnn_classify filter support to the ONNX Runtime DNN backend, enabling classification inference with ONNX models. >From 389c64703340a7d35443d2d7a42a168f14054a1c Mon Sep 17 00:00:00 2001 From: Dong Nguyen <[email protected]> Date: Wed, 2 Sep 2026 14:37:26 -0400 Subject: [PATCH] avfilter/dnn: add dnn_classify support to ONNX Runtime backend Add dnn_classify filter support to the ONNX Runtime DNN backend, enabling classification inference with ONNX models. --- libavfilter/dnn/dnn_backend_onnx.c | 126 +++++++++++++++++++++++++---- libavfilter/vf_dnn_classify.c | 5 +- 2 files changed, 115 insertions(+), 16 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_onnx.c b/libavfilter/dnn/dnn_backend_onnx.c index 009387b36d..34347be112 100644 --- a/libavfilter/dnn/dnn_backend_onnx.c +++ b/libavfilter/dnn/dnn_backend_onnx.c @@ -25,6 +25,7 @@ #include "libavutil/opt.h" #include "libavutil/avassert.h" +#include "libavutil/detection_bbox.h" #include "libavutil/imgutils.h" #include "libavutil/mem.h" #include "libavutil/avstring.h" @@ -97,25 +98,106 @@ static void init_ort_api(void) } \ } while (0) -static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue) +static int contain_valid_detection_bbox(AVFrame *frame) +{ + AVFrameSideData *sd; + const AVDetectionBBoxHeader *header; + const AVDetectionBBox *bbox; + + sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES); + if (!sd) + return 0; + + if (!sd->size) + return 0; + + header = (const AVDetectionBBoxHeader *)sd->data; + if (!header->nb_bboxes) + return 0; + + for (uint32_t i = 0; i < header->nb_bboxes; i++) { + bbox = av_get_detection_bbox(header, i); + if (bbox->x < 0 || bbox->w < 0 || bbox->x + bbox->w >= frame->width) + return 0; + if (bbox->y < 0 || bbox->h < 0 || bbox->y + bbox->h >= frame->height) + return 0; + if (bbox->classify_count == AV_NUM_DETECTION_BBOX_CLASSIFY) + return 0; + } + + return 1; +} + +static int extract_lltask_from_task(DNNFunctionType func_type, TaskItem *task, + Queue *lltask_queue, DNNExecBaseParams *exec_params) { ONNXModel *onnx_model = (ONNXModel *)task->model; DnnContext *ctx = onnx_model->ctx; - LastLevelTaskItem *lltask = av_malloc(sizeof(*lltask)); - if (!lltask) { - av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for LastLevelTaskItem\n"); - return AVERROR(ENOMEM); + switch (func_type) { + case DFT_PROCESS_FRAME: + case DFT_ANALYTICS_DETECT: + { + LastLevelTaskItem *lltask = av_malloc(sizeof(*lltask)); + if (!lltask) { + av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for LastLevelTaskItem\n"); + return AVERROR(ENOMEM); + } + task->inference_todo = 1; + task->inference_done = 0; + lltask->task = task; + if (ff_queue_push_back(lltask_queue, lltask) < 0) { + av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n"); + av_freep(&lltask); + return AVERROR(ENOMEM); + } + return 0; } - task->inference_todo = 1; - task->inference_done = 0; - lltask->task = task; - if (ff_queue_push_back(lltask_queue, lltask) < 0) { - av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n"); - av_freep(&lltask); - return AVERROR(ENOMEM); + case DFT_ANALYTICS_CLASSIFY: + { + const AVDetectionBBoxHeader *header; + AVFrame *frame = task->in_frame; + AVFrameSideData *sd; + DNNExecClassificationParams *params = (DNNExecClassificationParams *)exec_params; + + task->inference_todo = 0; + task->inference_done = 0; + + if (!contain_valid_detection_bbox(frame)) + return 0; + + sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES); + header = (const AVDetectionBBoxHeader *)sd->data; + + for (uint32_t i = 0; i < header->nb_bboxes; i++) { + LastLevelTaskItem *lltask; + const AVDetectionBBox *bbox = av_get_detection_bbox(header, i); + + if (params->target) { + if (av_strncasecmp(bbox->detect_label, params->target, sizeof(bbox->detect_label)) != 0) + continue; + } + + lltask = av_malloc(sizeof(*lltask)); + if (!lltask) { + av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for LastLevelTaskItem\n"); + return AVERROR(ENOMEM); + } + task->inference_todo++; + lltask->task = task; + lltask->bbox_index = i; + if (ff_queue_push_back(lltask_queue, lltask) < 0) { + av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n"); + av_freep(&lltask); + return AVERROR(ENOMEM); + } + } + return 0; + } + default: + avpriv_report_missing_feature(ctx, "model function type %d", func_type); + return AVERROR(ENOSYS); } - return 0; } static void onnx_free_request(ONNXInferRequest *request) @@ -435,6 +517,11 @@ static int fill_model_input_onnx(ONNXModel *onnx_model, ONNXRequestItem *request if (ret < 0) goto err; break; + case DFT_ANALYTICS_CLASSIFY: + ret = ff_frame_to_dnn_classify(task->in_frame, &input, lltask->bbox_index, ctx); + if (ret < 0) + goto err; + break; default: avpriv_report_missing_feature(ctx, "model function type %d", onnx_model->model.func_type); ret = AVERROR(ENOSYS); @@ -715,6 +802,15 @@ static void infer_completion_callback(void *args) if (ret < 0) goto err; break; + case DFT_ANALYTICS_CLASSIFY: + if (!onnx_model->model.classify_post_proc) { + av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide classify_post_proc\n"); + goto err; + } + onnx_model->model.classify_post_proc(task->in_frame, outputs, + request->lltask->bbox_index, + onnx_model->model.filter_ctx); + break; default: avpriv_report_missing_feature(ctx, "model function type %d", onnx_model->model.func_type); goto err; @@ -801,7 +897,7 @@ static int get_output_onnx(DNNModel *model, const char *input_name, int input_wi goto err; } - ret = extract_lltask_from_task(&task, onnx_model->lltask_queue); + ret = extract_lltask_from_task(DFT_PROCESS_FRAME, &task, onnx_model->lltask_queue, NULL); if (ret != 0) { av_log(ctx, AV_LOG_ERROR, "Unable to extract last level task from task.\n"); goto err; @@ -1174,7 +1270,7 @@ static int dnn_execute_model_onnx(const DNNModel *model, DNNExecBaseParams *exec return ret; } - ret = extract_lltask_from_task(task, onnx_model->lltask_queue); + ret = extract_lltask_from_task(model->func_type, task, onnx_model->lltask_queue, exec_params); if (ret != 0) { av_log(ctx, AV_LOG_ERROR, "Unable to extract last level task from task.\n"); return ret; diff --git a/libavfilter/vf_dnn_classify.c b/libavfilter/vf_dnn_classify.c index f92c41ab76..8faf6685ee 100644 --- a/libavfilter/vf_dnn_classify.c +++ b/libavfilter/vf_dnn_classify.c @@ -48,6 +48,9 @@ static const AVOption dnn_classify_options[] = { { "dnn_backend", "DNN backend", OFFSET(backend_type), AV_OPT_TYPE_INT, { .i64 = DNN_OV }, INT_MIN, INT_MAX, FLAGS, .unit = "backend" }, #if (CONFIG_LIBOPENVINO == 1) { "openvino", "openvino backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = DNN_OV }, 0, 0, FLAGS, .unit = "backend" }, +#endif +#if (CONFIG_LIBONNXRUNTIME == 1) + { "onnx", "onnx backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = DNN_ONNX }, 0, 0, FLAGS, .unit = "backend" }, #endif { "confidence", "threshold of confidence", OFFSET2(confidence), AV_OPT_TYPE_FLOAT, { .dbl = 0.5 }, 0, 1, FLAGS}, { "labels", "path to labels file", OFFSET2(labels_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, @@ -55,7 +58,7 @@ static const AVOption dnn_classify_options[] = { { NULL } }; -AVFILTER_DNN_DEFINE_CLASS(dnn_classify, DNN_OV); +AVFILTER_DNN_DEFINE_CLASS(dnn_classify, DNN_OV | DNN_ONNX); static int dnn_classify_post_proc(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx) { -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
