PR #23783 opened by michaelni URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23783 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23783.patch
avfilter/vf_hqdn3d: support dynamic frame sizes 3 minutes ago avfilter/vf_hqdn3d: reject unsupported frame parameter changes Fixes: out of array access Fixes: 9aj_hqdn3d_dynamic_res.mjpg / 9aj_generate_hqdn3d_dynamic_res_mjpg.py Fixes: wWDsy2oDvMuR Found-by: Adrian Junge (vurlo) <[email protected]> >From 42a956f757ca8a8e169d2d0081854ccef6c0e648 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sun, 12 Jul 2026 13:05:07 +0200 Subject: [PATCH 1/2] avfilter/vf_hqdn3d: reject unsupported frame parameter changes Fixes: out of array access Fixes: 9aj_hqdn3d_dynamic_res.mjpg / 9aj_generate_hqdn3d_dynamic_res_mjpg.py Fixes: wWDsy2oDvMuR Found-by: Adrian Junge (vurlo) <[email protected]> --- libavfilter/vf_hqdn3d.c | 34 +++++++++++++++++++++++++--------- libavfilter/vf_hqdn3d.h | 2 ++ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/libavfilter/vf_hqdn3d.c b/libavfilter/vf_hqdn3d.c index d880c2bdda..70a2ec4628 100644 --- a/libavfilter/vf_hqdn3d.c +++ b/libavfilter/vf_hqdn3d.c @@ -163,12 +163,8 @@ static int denoise_depth(HQDN3DContext *s, case 14: ret = denoise_depth(__VA_ARGS__, 14); break; \ case 16: ret = denoise_depth(__VA_ARGS__, 16); break; \ } \ - if (ret < 0) { \ - av_frame_free(&out); \ - if (!direct) \ - av_frame_free(&in); \ + if (ret < 0) \ return ret; \ - } \ } while (0) static void precalc_coefs(double dist25, int depth, int16_t *ct) @@ -281,12 +277,15 @@ static int config_input(AVFilterLink *inlink) ff_hqdn3d_init_x86(s); #endif + s->format = inlink->format; + s->width = inlink->w; + s->height = inlink->h; + return 0; } typedef struct ThreadData { AVFrame *in, *out; - int direct; } ThreadData; static int do_denoise(AVFilterContext *ctx, void *data, int job_nr, int n_jobs) @@ -295,7 +294,6 @@ static int do_denoise(AVFilterContext *ctx, void *data, int job_nr, int n_jobs) const ThreadData *td = data; AVFrame *out = td->out; AVFrame *in = td->in; - int direct = td->direct; denoise(s, in->data[job_nr], out->data[job_nr], s->line[job_nr], &s->frame_prev[job_nr], @@ -312,10 +310,21 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) { AVFilterContext *ctx = inlink->dst; AVFilterLink *outlink = ctx->outputs[0]; + HQDN3DContext *s = ctx->priv; AVFrame *out; int direct = av_frame_is_writable(in) && !ctx->is_disabled; ThreadData td; + int ret[3]; + + if (in->format != s->format || + in->width != s->width || + in->height != s->height) { + av_log(ctx, AV_LOG_ERROR, + "Frame size or format changed without filter graph reinitialization\n"); + av_frame_free(&in); + return AVERROR(EINVAL); + } if (direct) { out = in; @@ -331,9 +340,16 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) td.in = in; td.out = out; - td.direct = direct; /* one thread per plane */ - ff_filter_execute(ctx, do_denoise, &td, NULL, 3); + ff_filter_execute(ctx, do_denoise, &td, ret, 3); + for (int i = 0; i < FF_ARRAY_ELEMS(ret); i++) { + if (ret[i] < 0) { + av_frame_free(&out); + if (!direct) + av_frame_free(&in); + return ret[i]; + } + } if (ctx->is_disabled) { av_frame_free(&out); diff --git a/libavfilter/vf_hqdn3d.h b/libavfilter/vf_hqdn3d.h index 3279bbcc77..3467f27145 100644 --- a/libavfilter/vf_hqdn3d.h +++ b/libavfilter/vf_hqdn3d.h @@ -36,6 +36,8 @@ typedef struct HQDN3DContext { double strength[4]; int hsub, vsub; int depth; + int width, height; + enum AVPixelFormat format; void (*denoise_row[17])(uint8_t *src, uint8_t *dst, uint16_t *line_ant, uint16_t *frame_ant, ptrdiff_t w, int16_t *spatial, int16_t *temporal); } HQDN3DContext; -- 2.52.0 >From 3e28012d325c493fb5c0cfe5dc0780b2115eeb17 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sun, 12 Jul 2026 13:05:33 +0200 Subject: [PATCH 2/2] avfilter/vf_hqdn3d: support dynamic frame sizes --- libavfilter/avfilter.c | 3 ++- libavfilter/vf_hqdn3d.c | 21 ++++++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/libavfilter/avfilter.c b/libavfilter/avfilter.c index 0d46d303de..d10836df3b 100644 --- a/libavfilter/avfilter.c +++ b/libavfilter/avfilter.c @@ -1078,7 +1078,8 @@ int ff_filter_frame(AVFilterLink *link, AVFrame *frame) strcmp(link->dst->filter->name, "idet") && strcmp(link->dst->filter->name, "null") && strcmp(link->dst->filter->name, "scale") && - strcmp(link->dst->filter->name, "libplacebo")) { + strcmp(link->dst->filter->name, "libplacebo") && + strcmp(link->dst->filter->name, "hqdn3d")) { av_assert1(frame->format == link->format); av_assert1(frame->width == link->w); av_assert1(frame->height == link->h); diff --git a/libavfilter/vf_hqdn3d.c b/libavfilter/vf_hqdn3d.c index 70a2ec4628..98c05e886c 100644 --- a/libavfilter/vf_hqdn3d.c +++ b/libavfilter/vf_hqdn3d.c @@ -315,21 +315,28 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) AVFrame *out; int direct = av_frame_is_writable(in) && !ctx->is_disabled; ThreadData td; - int ret[3]; + int err, ret[3]; - if (in->format != s->format || - in->width != s->width || - in->height != s->height) { - av_log(ctx, AV_LOG_ERROR, - "Frame size or format changed without filter graph reinitialization\n"); + if (in->format != s->format) { av_frame_free(&in); return AVERROR(EINVAL); } + if (in->width != s->width || in->height != s->height) { + inlink->w = in->width; + inlink->h = in->height; + if ((err = config_input(inlink)) < 0) { + av_frame_free(&in); + return err; + } + outlink->w = in->width; + outlink->h = in->height; + } + if (direct) { out = in; } else { - out = ff_get_video_buffer(outlink, outlink->w, outlink->h); + out = ff_get_video_buffer(outlink, in->width, in->height); if (!out) { av_frame_free(&in); return AVERROR(ENOMEM); -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
