PR #24154 opened by Timo Rothenpieler (BtbN) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24154 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24154.patch
This should fix the crash/malfunction observed in #24126 The filter would run intermediate passes even when none were needed. Likewise, a passthrough-pass would not setup any passes, but not disable filtering, resulting in the same. Also fixes a bunch of things I ran into while working on it, like the filtering logic not taking chroma subsampling into account for what it considers downscaling. >From 1a29cba1520e01e05eacea57770331ca3ece8a93 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 22:47:47 +0200 Subject: [PATCH 1/5] avfilter/vf_scale_cuda: don't try to execute disabled filter passes --- libavfilter/vf_scale_cuda.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 240e40d43b..95ecde2774 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -157,6 +157,7 @@ typedef struct CUDAScaleContext { CUDAScaleFilter filters_uv[FILTER_NB]; CUDATex inter_tex; int use_filters; /* -1 for auto */ + int pass_x, pass_y; float param; } CUDAScaleContext; @@ -407,6 +408,8 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int s->frames_ctx = av_buffer_ref(inl->hw_frames_ctx); if (!s->frames_ctx) return AVERROR(ENOMEM); + + s->use_filters = 0; } else { s->passthrough = 0; @@ -451,9 +454,7 @@ static av_cold int cudascale_load_functions(AVFilterContext *ctx) if (s->use_filters) { /* Final pass is always vertical unless not vertically scaling */ - AVFilterLink *inlink = ctx->inputs[0]; - AVFilterLink *outlink = ctx->outputs[0]; - function_infix = inlink->h == outlink->h ? "Generic_h" : "Generic_v"; + function_infix = s->pass_y == FILTER_OUT ? "Generic_v" : "Generic_h"; s->interp_use_linear = 0; s->interp_as_integer = 0; } else { @@ -623,19 +624,19 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) if (ret < 0) return ret; - int pass_x = -1, pass_y = -1; + s->pass_x = s->pass_y = -1; if (inlink->w != outlink->w && inlink->h != outlink->h) { /* Always perform the horizontal scaling pass first */ - pass_x = FILTER_TMP; - pass_y = FILTER_OUT; + s->pass_x = FILTER_TMP; + s->pass_y = FILTER_OUT; } else if (inlink->w != outlink->w) { - pass_x = FILTER_OUT; + s->pass_x = FILTER_OUT; } else if (inlink->h != outlink->h) { - pass_y = FILTER_OUT; + s->pass_y = FILTER_OUT; } - if (pass_x >= 0) { - ret = cudascale_filter_init(ctx, &s->filters[pass_x], + if (s->pass_x >= 0) { + ret = cudascale_filter_init(ctx, &s->filters[s->pass_x], inlink->w, outlink->w, 0.0); if (ret < 0) goto fail; @@ -643,15 +644,15 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) const int src_size = AV_CEIL_RSHIFT(inlink->w, in_sub_x); const int dst_size = AV_CEIL_RSHIFT(outlink->w, out_sub_x); const double virtual_size = (double) outlink->w / (1 << out_sub_x); - ret = cudascale_filter_init(ctx, &s->filters_uv[pass_x], + ret = cudascale_filter_init(ctx, &s->filters_uv[s->pass_x], src_size, dst_size, virtual_size); if (ret < 0) goto fail; } } - if (pass_y >= 0) { - ret = cudascale_filter_init(ctx, &s->filters[pass_y], + if (s->pass_y >= 0) { + ret = cudascale_filter_init(ctx, &s->filters[s->pass_y], inlink->h, outlink->h, 0.0); if (ret < 0) goto fail; @@ -659,7 +660,7 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) const int src_size = AV_CEIL_RSHIFT(inlink->h, in_sub_y); const int dst_size = AV_CEIL_RSHIFT(outlink->h, out_sub_y); const double virtual_size = (double) outlink->h / (1 << out_sub_y); - ret = cudascale_filter_init(ctx, &s->filters_uv[pass_y], + ret = cudascale_filter_init(ctx, &s->filters_uv[s->pass_y], src_size, dst_size, virtual_size); if (ret < 0) goto fail; @@ -915,7 +916,7 @@ static int cudascale_scale(AVFilterContext *ctx, AVFrame *out, AVFrame *in) goto fail; const CUDATex *src = &in_tex; - if (s->use_filters) { + if (s->pass_x == FILTER_TMP) { /* Handle first pass separately */ s->inter_tex.color_range = in->color_range; ret = scalecuda_resize(ctx, FILTER_TMP, &s->inter_tex, src); -- 2.52.0 >From 403d17af1a95c21929be8a6e4761c9d075c36e5b Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 22:56:20 +0200 Subject: [PATCH 2/5] avfilter/vf_scale_cuda: skip allocating intermediate texture if not needed --- libavfilter/vf_scale_cuda.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 95ecde2774..2c22ec1aa4 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -667,9 +667,11 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) } } - ret = inter_buf_init(ctx, outlink->w, inlink->h); - if (ret < 0) - goto fail; + if (s->pass_x == FILTER_TMP) { + ret = inter_buf_init(ctx, outlink->w, inlink->h); + if (ret < 0) + goto fail; + } ret = 0; -- 2.52.0 >From 11042b0d0c7958b0a10a66afea40fd5202d020fd Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 23:13:16 +0200 Subject: [PATCH 3/5] avfilter/vf_scale_cuda: extract pass determination into its own function --- libavfilter/vf_scale_cuda.c | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 2c22ec1aa4..fc3a91d299 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -369,6 +369,27 @@ static av_cold void set_format_info(AVFilterContext *ctx, enum AVPixelFormat in_ } } +static av_cold void cudascale_setup_passes(AVFilterContext *ctx) +{ + CUDAScaleContext *s = ctx->priv; + AVFilterLink *inlink = ctx->inputs[0]; + AVFilterLink *outlink = ctx->outputs[0]; + + s->pass_x = s->pass_y = -1; + if (!s->use_filters) + return; + + if (inlink->w != outlink->w && inlink->h != outlink->h) { + /* Always perform the horizontal scaling pass first */ + s->pass_x = FILTER_TMP; + s->pass_y = FILTER_OUT; + } else if (inlink->w != outlink->w) { + s->pass_x = FILTER_OUT; + } else if (inlink->h != outlink->h) { + s->pass_y = FILTER_OUT; + } +} + static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int in_height, int out_width, int out_height) { @@ -433,6 +454,8 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int if (!outl->hw_frames_ctx) return AVERROR(ENOMEM); + cudascale_setup_passes(ctx); + return 0; } @@ -624,17 +647,6 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) if (ret < 0) return ret; - s->pass_x = s->pass_y = -1; - if (inlink->w != outlink->w && inlink->h != outlink->h) { - /* Always perform the horizontal scaling pass first */ - s->pass_x = FILTER_TMP; - s->pass_y = FILTER_OUT; - } else if (inlink->w != outlink->w) { - s->pass_x = FILTER_OUT; - } else if (inlink->h != outlink->h) { - s->pass_y = FILTER_OUT; - } - if (s->pass_x >= 0) { ret = cudascale_filter_init(ctx, &s->filters[s->pass_x], inlink->w, outlink->w, 0.0); -- 2.52.0 >From f55ab33a1c8cbf7943f6e21f0ba9fcf881ec4858 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 23:26:14 +0200 Subject: [PATCH 4/5] avfilter/vf_scale_cuda: always have valid passes setup --- libavfilter/vf_scale_cuda.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index fc3a91d299..7b001c9f5b 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -385,7 +385,7 @@ static av_cold void cudascale_setup_passes(AVFilterContext *ctx) s->pass_y = FILTER_OUT; } else if (inlink->w != outlink->w) { s->pass_x = FILTER_OUT; - } else if (inlink->h != outlink->h) { + } else { s->pass_y = FILTER_OUT; } } -- 2.52.0 >From 80c5c1ccb382d4274002ad0c000ce5f19da069d6 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 23:52:00 +0200 Subject: [PATCH 5/5] avfilter/vf_scale_cuda: consider chroma subsampling for scaling decisions --- libavfilter/vf_scale_cuda.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 7b001c9f5b..36c53288be 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -379,11 +379,16 @@ static av_cold void cudascale_setup_passes(AVFilterContext *ctx) if (!s->use_filters) return; - if (inlink->w != outlink->w && inlink->h != outlink->h) { + const int scale_x = inlink->w != outlink->w || + s->in_desc->log2_chroma_w != s->out_desc->log2_chroma_w; + const int scale_y = inlink->h != outlink->h || + s->in_desc->log2_chroma_h != s->out_desc->log2_chroma_h; + + if (scale_x && scale_y) { /* Always perform the horizontal scaling pass first */ s->pass_x = FILTER_TMP; s->pass_y = FILTER_OUT; - } else if (inlink->w != outlink->w) { + } else if (scale_x) { s->pass_x = FILTER_OUT; } else { s->pass_y = FILTER_OUT; @@ -444,7 +449,11 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int if (s->interp_algo == INTERP_ALGO_NEAREST) { s->use_filters = 0; - } else if (s->use_filters < 0 && (out_width < in_width || out_height < in_height)) + } else if (s->use_filters < 0 && ( + out_width < in_width || out_height < in_height || + AV_CEIL_RSHIFT(out_width, s->out_desc->log2_chroma_w) < AV_CEIL_RSHIFT(in_width, s->in_desc->log2_chroma_w) || + AV_CEIL_RSHIFT(out_height, s->out_desc->log2_chroma_h) < AV_CEIL_RSHIFT(in_height, s->in_desc->log2_chroma_h) + )) s->use_filters = 1; /* downscaling; needed for anti-aliasing */ else if (s->use_filters < 0) s->use_filters = 0; -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
