PR #25416 opened by michaelni URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25416 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25416.patch
>From 5eeccbc0512ae861f6a2aae99c1232026929c9c1 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 24 Oct 2026 05:46:56 +0200 Subject: [PATCH 1/8] avcodec/libsvtav1: unreference the input frame when eb_send_frame() fails eb_receive_packet() moves the next input frame into svt_enc->frame but returned without unreferencing it when eb_send_frame() failed, for example when Dolby Vision RPU coding is enabled and a frame carries no AV_FRAME_DATA_DOVI_METADATA. The next call moves another frame over the still referenced one, which leaks its buffers, or aborts in av_frame_move_ref() when assertions are enabled. Fixes: memleak Fixes: assertion failure Fixes: 3Vs9X1NJfz1H (AISLE-2026-0150-00864) Memleak Replicated through API with ASAN Assertion failure in av_frame_move_ref() Replicated through API Found-by: Joshua Rogers <[email protected]> --- libavcodec/libsvtav1.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/libsvtav1.c b/libavcodec/libsvtav1.c index 0bcc176243d..1e5732f45d6 100644 --- a/libavcodec/libsvtav1.c +++ b/libavcodec/libsvtav1.c @@ -664,9 +664,9 @@ static int eb_receive_packet(AVCodecContext *avctx, AVPacket *pkt) frame = NULL; ret = eb_send_frame(avctx, frame); + av_frame_unref(svt_enc->frame); if (ret < 0) return ret; - av_frame_unref(svt_enc->frame); svt_ret = svt_av1_enc_get_packet(svt_enc->svt_handle, &headerPtr, svt_enc->eos_flag); if (svt_ret == EB_NoErrorEmptyQueue) -- 2.54.0 >From 450b4987c36ad36dcf9358bf9e174f86afa8fb95 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 24 Oct 2026 05:47:15 +0200 Subject: [PATCH 2/8] avcodec/libsvtjpegxsdec: check the dimensions before initializing the decoder Since the dimensions are now updated before the decoder is re-initialized, decoder_initialized is cleared when the old decoder is closed, so that a failed re-initialization cannot leave the closed decoder in use. Fixes: memleak Fixes: qEiTY2cyEBYu (AISLE-2026-0150-00638) Memory and thread leak Replicated through UnModified FFmpeg Found-by: Joshua Rogers <[email protected]> --- libavcodec/libsvtjpegxsdec.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/libavcodec/libsvtjpegxsdec.c b/libavcodec/libsvtjpegxsdec.c index 7a325c44464..8a80cd560d6 100644 --- a/libavcodec/libsvtjpegxsdec.c +++ b/libavcodec/libsvtjpegxsdec.c @@ -118,15 +118,6 @@ static int svt_jpegxs_dec_decode(AVCodecContext* avctx, AVFrame* picture, int* g if (!svt_dec->decoder_initialized || ret != avctx->pix_fmt || avctx->width != svt_dec->config.width || avctx->height != svt_dec->config.height) { - if (svt_dec->decoder_initialized) - svt_jpeg_xs_decoder_close(&svt_dec->decoder); - err = svt_jpeg_xs_decoder_init(SVT_JPEGXS_API_VER_MAJOR, SVT_JPEGXS_API_VER_MINOR, - &svt_dec->decoder, avpkt->data, avpkt->size, &svt_dec->config); - if (err) { - av_log(avctx, AV_LOG_ERROR, "svt_jpeg_xs_decoder_init failed, err=%d\n", err); - return AVERROR_EXTERNAL; - } - avctx->pix_fmt = ret; ret = ff_set_dimensions(avctx, svt_dec->config.width, svt_dec->config.height); @@ -135,6 +126,16 @@ static int svt_jpegxs_dec_decode(AVCodecContext* avctx, AVFrame* picture, int* g return ret; } + if (svt_dec->decoder_initialized) + svt_jpeg_xs_decoder_close(&svt_dec->decoder); + svt_dec->decoder_initialized = 0; + err = svt_jpeg_xs_decoder_init(SVT_JPEGXS_API_VER_MAJOR, SVT_JPEGXS_API_VER_MINOR, + &svt_dec->decoder, avpkt->data, avpkt->size, &svt_dec->config); + if (err) { + av_log(avctx, AV_LOG_ERROR, "svt_jpeg_xs_decoder_init failed, err=%d\n", err); + return AVERROR_EXTERNAL; + } + svt_dec->decoder_initialized = 1; } -- 2.54.0 >From ddb84cde4a377a24a303040100f34689916a7596 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 24 Oct 2026 05:47:15 +0200 Subject: [PATCH 3/8] avcodec/liboapvenc: copy the input with the dimensions of the codec context Fixes: out of array write Fixes: HwSE6IYkyPvk (AISLE-2026-0150-00139) Out of array write Replicated through API with ASAN Found-by: Joshua Rogers <[email protected]> --- libavcodec/liboapvenc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/liboapvenc.c b/libavcodec/liboapvenc.c index 32a8cd5f1c9..0f59d65e146 100644 --- a/libavcodec/liboapvenc.c +++ b/libavcodec/liboapvenc.c @@ -582,7 +582,7 @@ static int liboapve_encode(AVCodecContext *avctx, AVPacket *avpkt, int ret; av_image_copy2((uint8_t **)imgb->a, imgb->s, frame->data, frame->linesize, - frame->format, frame->width, frame->height); + avctx->pix_fmt, avctx->width, avctx->height); imgb->ts[0] = frame->pts; -- 2.54.0 >From c888494506c2b38c1871592e7cc0c3856e5023e9 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 24 Oct 2026 05:47:15 +0200 Subject: [PATCH 4/8] avcodec/libx265: free the Dolby Vision RPU after encoding the picture Fixes: memleak Fixes: 3wvWUVmf56kN (AISLE-2026-0150-00168) Memleak Replicated through UnModified FFmpeg with ASAN Found-by: Joshua Rogers <[email protected]> --- libavcodec/libx265.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libavcodec/libx265.c b/libavcodec/libx265.c index c5e6b8c1501..0ec4f87f5fb 100644 --- a/libavcodec/libx265.c +++ b/libavcodec/libx265.c @@ -878,6 +878,9 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt, for (i = 0; i < sei->numPayloads; i++) av_free(sei->payloads[i].payload); av_freep(&x265pic.quantOffsets); +#if X265_BUILD >= 167 + av_freep(&x265pic.rpu.payload); +#endif if (ret < 0) return AVERROR_EXTERNAL; -- 2.54.0 >From 98445d7d3b91e6b5bb522b81cebe9b2bdf044734 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 24 Oct 2026 05:47:15 +0200 Subject: [PATCH 5/8] avcodec/libxevd: check the NAL unit length against the remaining packet data Fixes: out of array read Fixes: 5sHdL1UwYoZ2 (AISLE-2026-0150-00006) Out of array read Replicated through UnModified FFmpeg with ASAN Out of array read Replicated through API with ASAN Found-by: Joshua Rogers <[email protected]> --- libavcodec/libxevd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/libxevd.c b/libavcodec/libxevd.c index dd8ffb2f6ad..7db7ec6dc3a 100644 --- a/libavcodec/libxevd.c +++ b/libavcodec/libxevd.c @@ -348,7 +348,7 @@ static int libxevd_receive_frame(AVCodecContext *avctx, AVFrame *frame) memset(&stat, 0, sizeof(XEVD_STAT)); nalu_size = read_nal_unit_length(pkt_au->data + bs_read_pos, XEVD_NAL_UNIT_LENGTH_BYTE, avctx); - if (nalu_size == 0) { + if (nalu_size <= 0 || nalu_size > pkt_au->size - bs_read_pos - XEVD_NAL_UNIT_LENGTH_BYTE) { av_log(avctx, AV_LOG_ERROR, "Invalid bitstream\n"); av_packet_free(&pkt_au); ret = AVERROR_INVALIDDATA; -- 2.54.0 >From ad8e1a4d15ee6da8850b0faa89e3daeb43aaf68c Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 24 Oct 2026 05:47:37 +0200 Subject: [PATCH 6/8] avcodec/libxevd: return EAGAIN when an access unit produces no frame An access unit that produces no picture, such as one holding only parameter sets or one of at most 4 bytes, left the NAL unit loop without a frame and returned 0, which the generic decode code takes for a decoded frame and rejects with an assertion failure. Return EAGAIN like the delayed frame path does. Fixes: assertion failure Fixes: XF1FdBsTa8zq (AISLE-2026-0150-00628) Assertion failure Replicated through UnModified FFmpeg Found-by: Joshua Rogers <[email protected]> --- libavcodec/libxevd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/libxevd.c b/libavcodec/libxevd.c index 7db7ec6dc3a..3913e223d01 100644 --- a/libavcodec/libxevd.c +++ b/libavcodec/libxevd.c @@ -458,7 +458,7 @@ static int libxevd_receive_frame(AVCodecContext *avctx, AVFrame *frame) } } - return ret; + return AVERROR(EAGAIN); } /** -- 2.54.0 >From b285ceb8a5159a27133db4d9ab1d1b4b108083ab Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 24 Oct 2026 05:48:38 +0200 Subject: [PATCH 7/8] avcodec/libxevd: free the access unit packet when no picture keeps it Packets stored in pictures that are still buffered when the decoder is closed are not covered by this and remain leaked, see ebin21hUoZTa. Fixes: memleak Fixes: AkIti8GalyUg (AISLE-2026-0150-00007) Memleak Replicated through UnModified FFmpeg with ASAN Found-by: Joshua Rogers <[email protected]> --- libavcodec/libxevd.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavcodec/libxevd.c b/libavcodec/libxevd.c index 3913e223d01..4aad67af6f1 100644 --- a/libavcodec/libxevd.c +++ b/libavcodec/libxevd.c @@ -331,6 +331,7 @@ static int libxevd_receive_frame(AVCodecContext *avctx, AVFrame *frame) if (pkt->size > 0) { int bs_read_pos = 0; + int pkt_au_retained = 0; XEVD_STAT stat; XEVD_BITB bitb; int nalu_size; @@ -394,6 +395,7 @@ static int libxevd_receive_frame(AVCodecContext *avctx, AVFrame *frame) // stat.fnum - has negative value if the decoded data is not frame if (stat.fnum >= 0) { + pkt_au_retained = 1; // store stat.stype into the bitb.pdata[1] that will be retrieved in an image *(int*)bitb.pdata[1] = stat.stype; @@ -427,6 +429,8 @@ static int libxevd_receive_frame(AVCodecContext *avctx, AVFrame *frame) } } } + if (!pkt_au_retained) + av_packet_free(&pkt_au); } else { // decoder draining mode handling while (1) { -- 2.54.0 >From 1b8f5b19c4883f42d1843b457d5958f58bcac862 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 24 Oct 2026 05:49:36 +0200 Subject: [PATCH 8/8] avcodec/libxevd: free the slice type storage of NAL units that complete no picture Found during triage of the security report AkIti8GalyUg. Fixes: memleak Fixes: 5nzyL324psH3 Memleak Replicated through UnModified FFmpeg with ASAN --- libavcodec/libxevd.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/libxevd.c b/libavcodec/libxevd.c index 4aad67af6f1..b1bede5e1bf 100644 --- a/libavcodec/libxevd.c +++ b/libavcodec/libxevd.c @@ -380,6 +380,8 @@ static int libxevd_receive_frame(AVCodecContext *avctx, AVFrame *frame) } bs_read_pos += nalu_size; + if (stat.fnum < 0) + av_freep(&bitb.pdata[1]); if (stat.nalu_type == XEVD_NUT_SPS) { // EVC stream parameters changed if ((ret = export_stream_params(xectx, avctx)) != 0) { -- 2.54.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
