PR #22655 opened by James Almer (jamrial) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22655 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22655.patch
Instead of having a single pool of frame buffers that will get uninitialized and replaced every time a decoder returns a frame with different properties, allow decoders to have more than one such pool. For example the case of H.26x and AV1, which may have >= 16 different concurrent sequence parameters, as well as post-processing enhancements like film-grain, LCEVC and Dolby Vision, which can result in frames with different properties than the input base. >From cfc3d80b52559686499f0deb7f05aef19b49dc0b Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Sun, 29 Mar 2026 16:38:22 -0300 Subject: [PATCH 1/7] avcodec/decode: remove code overwritting frame dimensions for LCEVC A better and cleaner approach will be implemented in the following commits. Signed-off-by: James Almer <[email protected]> --- libavcodec/decode.c | 68 +++++++++---------------------------------- libavcodec/lcevcdec.h | 7 +++++ 2 files changed, 20 insertions(+), 55 deletions(-) diff --git a/libavcodec/decode.c b/libavcodec/decode.c index 6ddf469663..4482de40b2 100644 --- a/libavcodec/decode.c +++ b/libavcodec/decode.c @@ -95,14 +95,7 @@ typedef struct DecodeContext { uint64_t side_data_pref_mask; #if CONFIG_LIBLCEVC_DEC - struct { - FFLCEVCContext *ctx; - int frame; - int base_width; - int base_height; - int width; - int height; - } lcevc; + FFLCEVCState lcevc; #endif } DecodeContext; @@ -1621,32 +1614,6 @@ int ff_decode_frame_props(AVCodecContext *avctx, AVFrame *frame) break; } -#if CONFIG_LIBLCEVC_DEC - AVCodecInternal *avci = avctx->internal; - DecodeContext *dc = decode_ctx(avci); - - dc->lcevc.frame = dc->lcevc.ctx && avctx->codec_type == AVMEDIA_TYPE_VIDEO && - av_frame_get_side_data(frame, AV_FRAME_DATA_LCEVC); - - if (dc->lcevc.frame) { - int ret = ff_lcevc_parse_frame(dc->lcevc.ctx, frame, - &dc->lcevc.width, &dc->lcevc.height, avctx); - if (ret < 0 && (avctx->err_recognition & AV_EF_EXPLODE)) - return ret; - - // force get_buffer2() to allocate the base frame using the same dimensions - // as the final enhanced frame, in order to prevent reinitializing the buffer - // pools unnecessarely - if (!ret && dc->lcevc.width && dc->lcevc.height) { - dc->lcevc.base_width = frame->width; - dc->lcevc.base_height = frame->height; - frame->width = dc->lcevc.width; - frame->height = dc->lcevc.height; - } else - dc->lcevc.frame = 0; - } -#endif - return 0; } @@ -1699,31 +1666,26 @@ int ff_attach_decode_data(AVCodecContext *avctx, AVFrame *frame) AVCodecInternal *avci = avctx->internal; DecodeContext *dc = decode_ctx(avci); - if (!dc->lcevc.frame) { - dc->lcevc.frame = dc->lcevc.ctx && avctx->codec_type == AVMEDIA_TYPE_VIDEO && - av_frame_get_side_data(frame, AV_FRAME_DATA_LCEVC); + dc->lcevc.present = dc->lcevc.ctx && avctx->codec_type == AVMEDIA_TYPE_VIDEO && + av_frame_get_side_data(frame, AV_FRAME_DATA_LCEVC); - if (dc->lcevc.frame) { - int ret = ff_lcevc_parse_frame(dc->lcevc.ctx, frame, - &dc->lcevc.width, &dc->lcevc.height, avctx); - if (ret < 0 && (avctx->err_recognition & AV_EF_EXPLODE)) + if (dc->lcevc.present) { + int ret = ff_lcevc_parse_frame(dc->lcevc.ctx, frame, + &dc->lcevc.width, &dc->lcevc.height, avctx); + if (ret < 0) { + if (avctx->err_recognition & AV_EF_EXPLODE) return ret; - - if (!ret && dc->lcevc.width && dc->lcevc.height) { - dc->lcevc.base_width = frame->width; - dc->lcevc.base_height = frame->height; - } else - dc->lcevc.frame = 0; + dc->lcevc.present = 0; } } - if (dc->lcevc.frame) { + + if (dc->lcevc.present) { FFLCEVCFrame *frame_ctx; int ret; - if (fdd->post_process || !dc->lcevc.width || !dc->lcevc.height) { - dc->lcevc.frame = 0; + dc->lcevc.present = 0; + if (fdd->post_process || !dc->lcevc.width || !dc->lcevc.height) return 0; - } frame_ctx = av_mallocz(sizeof(*frame_ctx)); if (!frame_ctx) @@ -1740,9 +1702,6 @@ int ff_attach_decode_data(AVCodecContext *avctx, AVFrame *frame) frame_ctx->frame->height = dc->lcevc.height; frame_ctx->frame->format = frame->format; - frame->width = dc->lcevc.base_width; - frame->height = dc->lcevc.base_height; - ret = avctx->get_buffer2(avctx, frame_ctx->frame, 0); if (ret < 0) { ff_lcevc_unref(frame_ctx); @@ -1755,7 +1714,6 @@ int ff_attach_decode_data(AVCodecContext *avctx, AVFrame *frame) fdd->post_process_opaque_free = ff_lcevc_unref; fdd->post_process = ff_lcevc_process; } - dc->lcevc.frame = 0; #endif return 0; diff --git a/libavcodec/lcevcdec.h b/libavcodec/lcevcdec.h index a65214344d..2b55bbf078 100644 --- a/libavcodec/lcevcdec.h +++ b/libavcodec/lcevcdec.h @@ -38,6 +38,13 @@ typedef struct FFLCEVCContext { int initialized; } FFLCEVCContext; +typedef struct FFLCEVCState { + FFLCEVCContext *ctx; + int present; + int width; + int height; +} FFLCEVCState; + struct AVFrame; typedef struct FFLCEVCFrame { -- 2.52.0 >From f99b78e3dfdd8cb5c2487ea9d3b0f4a50a7eeacd Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Sun, 29 Mar 2026 17:15:21 -0300 Subject: [PATCH 2/7] avcodec/get_buffer: support more than one pool of frame buffers Instead of having a single pool of frame buffers that will get uninitialized and replaced every time a decoder returns a frame with different properties, allow decoders to have more than one such pool. For example the case of H.26x and AV1, which may have >= 16 different concurrent sequence parameters, as well as post-processing enhancements like film-grain, LCEVC and Dolby Vision, which can result in frames with different properties than the input base. Signed-off-by: James Almer <[email protected]> --- libavcodec/avcodec.c | 8 ++++-- libavcodec/codec_internal.h | 8 ++++++ libavcodec/decode.c | 2 ++ libavcodec/get_buffer.c | 49 ++++++++++++++++++++++--------------- libavcodec/internal.h | 3 ++- libavcodec/pthread_frame.c | 12 +++++++-- 6 files changed, 57 insertions(+), 25 deletions(-) diff --git a/libavcodec/avcodec.c b/libavcodec/avcodec.c index 9ff4c7f319..d3187fac6d 100644 --- a/libavcodec/avcodec.c +++ b/libavcodec/avcodec.c @@ -202,10 +202,12 @@ int attribute_align_arg avcodec_open2(AVCodecContext *avctx, const AVCodec *code avci->buffer_frame = av_frame_alloc(); avci->buffer_pkt = av_packet_alloc(); - if (!avci->buffer_frame || !avci->buffer_pkt) { + avci->pool = av_calloc(FFMAX(1, codec2->pool_count), sizeof(*avci->pool)); + if (!avci->buffer_frame || !avci->buffer_pkt || !avci->pool) { ret = AVERROR(ENOMEM); goto free_and_end; } + avci->pool_count = FFMAX(1, codec2->pool_count); if (codec2->priv_data_size > 0) { if (!avctx->priv_data) { @@ -458,7 +460,9 @@ av_cold void ff_codec_close(AVCodecContext *avctx) av_frame_free(&avci->in_frame); av_frame_free(&avci->recon_frame); - av_refstruct_unref(&avci->pool); + for (i = 0; i < avci->pool_count; i++) + av_refstruct_unref(&avci->pool[i]); + av_freep(&avci->pool); av_refstruct_pool_uninit(&avci->progress_frame_pool); if (av_codec_is_decoder(avctx->codec)) ff_decode_internal_uninit(avctx); diff --git a/libavcodec/codec_internal.h b/libavcodec/codec_internal.h index eea982e56c..dc2bb6761f 100644 --- a/libavcodec/codec_internal.h +++ b/libavcodec/codec_internal.h @@ -285,6 +285,14 @@ typedef struct FFCodec { unsigned flags, const void **out_configs, int *out_num_configs); + + /** + * Maximum estimated amount of different frame prop variations a sane + * stream could have. Used to allocate buffer pools in avcodec_default_get_buffer2(). + * + * If unset, a default of 1 is used. + */ + unsigned int pool_count; } FFCodec; static av_always_inline const FFCodec *ffcodec(const AVCodec *codec) diff --git a/libavcodec/decode.c b/libavcodec/decode.c index 4482de40b2..06b82f72bc 100644 --- a/libavcodec/decode.c +++ b/libavcodec/decode.c @@ -2300,6 +2300,8 @@ av_cold void ff_decode_flush_buffers(AVCodecContext *avctx) av_packet_unref(avci->last_pkt_props); av_packet_unref(avci->in_pkt); + for (int i = 0; i < avci->pool_count; i++) + av_refstruct_unref(&avci->pool[i]); dc->pts_correction_last_pts = dc->pts_correction_last_dts = INT64_MIN; diff --git a/libavcodec/get_buffer.c b/libavcodec/get_buffer.c index b391adf24f..c96a79f74a 100644 --- a/libavcodec/get_buffer.c +++ b/libavcodec/get_buffer.c @@ -62,20 +62,30 @@ static void frame_pool_free(AVRefStructOpaque unused, void *obj) av_buffer_pool_uninit(&pool->pools[i]); } -static int update_frame_pool(AVCodecContext *avctx, AVFrame *frame) +static int get_frame_pool(AVCodecContext *avctx, FramePool **poolp, AVFrame *frame) { - FramePool *pool = avctx->internal->pool; - int i, ret; + FramePool *pool = NULL; + int i, index, ret; - if (pool && pool->format == frame->format) { - if (avctx->codec_type == AVMEDIA_TYPE_VIDEO && - pool->width == frame->width && pool->height == frame->height) - return 0; - if (avctx->codec_type == AVMEDIA_TYPE_AUDIO && - pool->channels == frame->ch_layout.nb_channels && - frame->nb_samples == pool->samples) - return 0; + for (index = 0; index < avctx->internal->pool_count; index++) { + pool = avctx->internal->pool[index]; + if (!pool) + break; + if (pool->format == frame->format) { + if (avctx->codec_type == AVMEDIA_TYPE_VIDEO && + pool->width == frame->width && pool->height == frame->height) { + *poolp = pool; + return 0; + } + if (avctx->codec_type == AVMEDIA_TYPE_AUDIO && + pool->channels == frame->ch_layout.nb_channels && + frame->nb_samples == pool->samples) { + *poolp = pool; + return 0; + } + } } + index %= avctx->internal->pool_count; pool = av_refstruct_alloc_ext(sizeof(*pool), 0, NULL, frame_pool_free); if (!pool) @@ -160,8 +170,8 @@ static int update_frame_pool(AVCodecContext *avctx, AVFrame *frame) default: av_assert0(0); } - av_refstruct_unref(&avctx->internal->pool); - avctx->internal->pool = pool; + av_refstruct_unref(&avctx->internal->pool[index]); + avctx->internal->pool[index] = *poolp = pool; return 0; fail: @@ -169,9 +179,8 @@ fail: return ret; } -static int audio_get_buffer(AVCodecContext *avctx, AVFrame *frame) +static int audio_get_buffer(AVCodecContext *avctx, FramePool *pool, AVFrame *frame) { - FramePool *pool = avctx->internal->pool; int planes = pool->planes; int i; @@ -214,9 +223,8 @@ fail: return AVERROR(ENOMEM); } -static int video_get_buffer(AVCodecContext *s, AVFrame *pic) +static int video_get_buffer(AVCodecContext *s, FramePool *pool, AVFrame *pic) { - FramePool *pool = s->internal->pool; int i; if (pic->data[0] || pic->data[1] || pic->data[2] || pic->data[3]) { @@ -252,6 +260,7 @@ fail: int avcodec_default_get_buffer2(AVCodecContext *avctx, AVFrame *frame, int flags) { + FramePool *pool; int ret; if (avctx->hw_frames_ctx) { @@ -277,14 +286,14 @@ int avcodec_default_get_buffer2(AVCodecContext *avctx, AVFrame *frame, int flags return ret; } - if ((ret = update_frame_pool(avctx, frame)) < 0) + if ((ret = get_frame_pool(avctx, &pool, frame)) < 0) return ret; switch (avctx->codec_type) { case AVMEDIA_TYPE_VIDEO: - return video_get_buffer(avctx, frame); + return video_get_buffer(avctx, pool, frame); case AVMEDIA_TYPE_AUDIO: - return audio_get_buffer(avctx, frame); + return audio_get_buffer(avctx, pool, frame); default: return -1; } diff --git a/libavcodec/internal.h b/libavcodec/internal.h index 137fd52745..4f35c4b5fb 100644 --- a/libavcodec/internal.h +++ b/libavcodec/internal.h @@ -66,7 +66,8 @@ typedef struct AVCodecInternal { */ int pad_samples; - struct FramePool *pool; + unsigned pool_count; + struct FramePool **pool; struct AVRefStructPool *progress_frame_pool; diff --git a/libavcodec/pthread_frame.c b/libavcodec/pthread_frame.c index c2853086ce..8d97c30e4e 100644 --- a/libavcodec/pthread_frame.c +++ b/libavcodec/pthread_frame.c @@ -402,7 +402,8 @@ FF_ENABLE_DEPRECATION_WARNINGS dst->hwaccel_flags = src->hwaccel_flags; - av_refstruct_replace(&dst->internal->pool, src->internal->pool); + for (int i = 0; i < src->internal->pool_count; i++) + av_refstruct_replace(&dst->internal->pool[i], dst->internal->pool[i]); ff_decode_internal_sync(dst, src); } @@ -782,7 +783,9 @@ av_cold void ff_frame_thread_free(AVCodecContext *avctx, int thread_count) av_freep(&ctx->priv_data); } - av_refstruct_unref(&ctx->internal->pool); + for (int j = 0; j < ctx->internal->pool_count; j++) + av_refstruct_unref(&ctx->internal->pool[j]); + av_freep(&ctx->internal->pool); av_packet_free(&ctx->internal->in_pkt); av_packet_free(&ctx->internal->last_pkt_props); ff_decode_internal_uninit(ctx); @@ -846,6 +849,11 @@ static av_cold int init_thread(PerThreadContext *p, int *threads_to_free, copy->internal->thread_ctx = p; copy->internal->progress_frame_pool = avctx->internal->progress_frame_pool; + copy->internal->pool = av_calloc(avctx->internal->pool_count, sizeof(*copy->internal->pool)); + if (!copy->internal->pool) + return AVERROR(ENOMEM); + copy->internal->pool_count = avctx->internal->pool_count; + copy->delay = avctx->delay; if (codec->priv_data_size) { -- 2.52.0 >From 8bc41c64650eccbb76bc1fb822c0f24b22549fb0 Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Sun, 29 Mar 2026 17:44:09 -0300 Subject: [PATCH 3/7] avcodec/av1dec: set FFCodec.pool_count Signed-off-by: James Almer <[email protected]> --- libavcodec/av1dec.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavcodec/av1dec.c b/libavcodec/av1dec.c index 1635cbea54..46bb085472 100644 --- a/libavcodec/av1dec.c +++ b/libavcodec/av1dec.c @@ -1639,6 +1639,7 @@ const FFCodec ff_av1_decoder = { .flush = av1_decode_flush, .p.profiles = NULL_IF_CONFIG_SMALL(ff_av1_profiles), .p.priv_class = &av1_class, + .pool_count = AV1_MAX_OPERATING_POINTS, .hw_configs = (const AVCodecHWConfigInternal *const []) { #if CONFIG_AV1_DXVA2_HWACCEL HWACCEL_DXVA2(av1), -- 2.52.0 >From 97aecaf556d7412394b77ba76b7562043aedbbaa Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Sun, 29 Mar 2026 17:42:23 -0300 Subject: [PATCH 4/7] avcodec/h264dec: set FFCodec.pool_count Signed-off-by: James Almer <[email protected]> --- libavcodec/h264dec.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavcodec/h264dec.c b/libavcodec/h264dec.c index 809a938386..51c69f0434 100644 --- a/libavcodec/h264dec.c +++ b/libavcodec/h264dec.c @@ -1117,6 +1117,7 @@ const FFCodec ff_h264_decoder = { .p.capabilities = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_SLICE_THREADS | AV_CODEC_CAP_FRAME_THREADS, + .pool_count = H264_MAX_SPS_COUNT, .hw_configs = (const AVCodecHWConfigInternal *const []) { #if CONFIG_H264_DXVA2_HWACCEL HWACCEL_DXVA2(h264), -- 2.52.0 >From 52f6bcee772790e59e651b0d44f9786577a287c4 Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Sun, 29 Mar 2026 17:42:37 -0300 Subject: [PATCH 5/7] avcodec/hevc/hevcdec: set FFCodec.pool_count Signed-off-by: James Almer <[email protected]> --- libavcodec/hevc/hevcdec.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavcodec/hevc/hevcdec.c b/libavcodec/hevc/hevcdec.c index ae5f0fe69d..c896bcb262 100644 --- a/libavcodec/hevc/hevcdec.c +++ b/libavcodec/hevc/hevcdec.c @@ -4253,6 +4253,7 @@ const FFCodec ff_hevc_decoder = { FF_CODEC_CAP_USES_PROGRESSFRAMES | FF_CODEC_CAP_INIT_CLEANUP, .p.profiles = NULL_IF_CONFIG_SMALL(ff_hevc_profiles), + .pool_count = HEVC_MAX_SPS_COUNT, .hw_configs = (const AVCodecHWConfigInternal *const []) { #if CONFIG_HEVC_DXVA2_HWACCEL HWACCEL_DXVA2(hevc), -- 2.52.0 >From f8370b75d3d4c9325239c181970b42e76ca8c108 Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Sun, 29 Mar 2026 17:43:02 -0300 Subject: [PATCH 6/7] avcodec/libdav1d: set FFCodec.pool_count Signed-off-by: James Almer <[email protected]> --- libavcodec/libdav1d.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavcodec/libdav1d.c b/libavcodec/libdav1d.c index d9755a45b2..1f92c6af18 100644 --- a/libavcodec/libdav1d.c +++ b/libavcodec/libdav1d.c @@ -719,6 +719,7 @@ const FFCodec ff_libdav1d_decoder = { .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_OTHER_THREADS, .caps_internal = FF_CODEC_CAP_SETS_FRAME_PROPS | FF_CODEC_CAP_AUTO_THREADS, + .pool_count = AV1_MAX_OPERATING_POINTS, .p.priv_class = &libdav1d_class, .p.wrapper_name = "libdav1d", }; -- 2.52.0 >From 8d3c982efbc8db44d53897280a2af17cf15621af Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Sun, 29 Mar 2026 17:42:48 -0300 Subject: [PATCH 7/7] avcodec/vvc/dec: set FFCodec.pool_count Signed-off-by: James Almer <[email protected]> --- libavcodec/vvc/dec.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavcodec/vvc/dec.c b/libavcodec/vvc/dec.c index 194e2fc7ef..9cdcb0b3d2 100644 --- a/libavcodec/vvc/dec.c +++ b/libavcodec/vvc/dec.c @@ -1319,6 +1319,7 @@ const FFCodec ff_vvc_decoder = { .caps_internal = FF_CODEC_CAP_EXPORTS_CROPPING | FF_CODEC_CAP_INIT_CLEANUP | FF_CODEC_CAP_AUTO_THREADS, .p.profiles = NULL_IF_CONFIG_SMALL(ff_vvc_profiles), + .pool_count = VVC_MAX_SPS_COUNT, .hw_configs = (const AVCodecHWConfigInternal *const []) { #if CONFIG_VVC_VAAPI_HWACCEL HWACCEL_VAAPI(vvc), -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
