PR #25573 opened by michaelni URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25573 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25573.patch
>From 36546b8a4d1629beeee0558d86d250f190af9fe2 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 24 Oct 2026 06:38:34 +0200 Subject: [PATCH 1/5] avcodec/proresenc_kostya: compute the frame size upper bound in 64 bits Signed integer overflow Replicated through UnModified FFmpeg with UBSAN Out of array write Replicated through UnModified FFmpeg with ASAN Fixes: out of array write Fixes: EMbkV19WJjmA Fixes: AISLE-2026-0150-00821 Found-by: Joshua Rogers (AISLE) Signed-off-by: Michael Niedermayer <[email protected]> --- libavcodec/proresenc_kostya.c | 5 +---- libavcodec/proresenc_kostya_common.c | 19 ++++++++++++++----- libavcodec/proresenc_kostya_common.h | 1 + 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/libavcodec/proresenc_kostya.c b/libavcodec/proresenc_kostya.c index 65346ca1700..bf7c4bda43a 100644 --- a/libavcodec/proresenc_kostya.c +++ b/libavcodec/proresenc_kostya.c @@ -762,12 +762,9 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt, int frame_size, picture_size, slice_size; int pkt_size, ret; int max_slice_size = (ctx->frame_size_upper_bound - 200) / (ctx->pictures_per_frame * ctx->slices_per_picture + 1); - int slice_size_upper_bound = ctx->mbs_per_slice * (ctx->chroma_factor == CFACTOR_Y444 ? 768 : 512) * 8; + int slice_size_upper_bound = ctx->slice_size_upper_bound; uint8_t frame_flags; - if (ctx->alpha_bits) - slice_size_upper_bound += ctx->mbs_per_slice * 256 * (1 + ctx->alpha_bits + 1) + 7 >> 3; - ctx->pic = pic; pkt_size = ctx->frame_size_upper_bound + slice_size_upper_bound; diff --git a/libavcodec/proresenc_kostya_common.c b/libavcodec/proresenc_kostya_common.c index 558637d144a..173268fff84 100644 --- a/libavcodec/proresenc_kostya_common.c +++ b/libavcodec/proresenc_kostya_common.c @@ -24,6 +24,7 @@ #include "libavutil/pixdesc.h" #include "avcodec.h" #include "bytestream.h" +#include "encode.h" #include "proresdata.h" #include <sys/types.h> #include "proresenc_kostya_common.h" @@ -283,20 +284,28 @@ av_cold int ff_prores_kostya_encode_init(AVCodecContext *avctx, ProresContext *c ctx->bits_per_mb += ls_chroma * 4; } - ctx->frame_size_upper_bound = (ctx->pictures_per_frame * + int64_t frame_size_upper_bound = (int64_t)(ctx->pictures_per_frame * ctx->slices_per_picture + 1) * (2 + 2 * ctx->num_planes + (mps * ctx->bits_per_mb) / 8) + 200; + ctx->slice_size_upper_bound = mps * (ctx->chroma_factor == CFACTOR_Y444 ? 768 : 512) * 8; if (ctx->alpha_bits) { // The alpha plane is run-coded and might exceed the bit budget. - ctx->frame_size_upper_bound += (ctx->pictures_per_frame * - ctx->slices_per_picture + 1) * - /* num pixels per slice */ (ctx->mbs_per_slice * 256 * - /* bits per pixel */ (1 + ctx->alpha_bits + 1) + 7 >> 3); + int alpha_slice_size = /* num pixels per slice */ mps * 256 * + /* bits per pixel */ (1 + ctx->alpha_bits + 1) + 7 >> 3; + frame_size_upper_bound += (int64_t)(ctx->pictures_per_frame * + ctx->slices_per_picture + 1) * alpha_slice_size; + ctx->slice_size_upper_bound += alpha_slice_size; } + if (frame_size_upper_bound + ctx->slice_size_upper_bound + FF_INPUT_BUFFER_MIN_SIZE > INT_MAX) { + av_log(avctx, AV_LOG_ERROR, "Frame too large to encode\n"); + return AVERROR(EINVAL); + } + ctx->frame_size_upper_bound = frame_size_upper_bound; + avctx->codec_tag = ctx->profile_info->tag; avctx->profile = ctx->profile; diff --git a/libavcodec/proresenc_kostya_common.h b/libavcodec/proresenc_kostya_common.h index ed5355db0fb..8055394a723 100644 --- a/libavcodec/proresenc_kostya_common.h +++ b/libavcodec/proresenc_kostya_common.h @@ -112,6 +112,7 @@ typedef struct ProresContext { int quant_sel; int frame_size_upper_bound; + int slice_size_upper_bound; int profile; const struct prores_profile *profile_info; -- 2.54.0 >From 4c62d7561af4a98ee9219e5f60554e81e6a00623 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 24 Oct 2026 06:38:34 +0200 Subject: [PATCH 2/5] avcodec/huffyuvenc: reject frames too small for the median predictor SEGV Replicated through UnModified FFmpeg with ASAN (width 2) Out of array read Replicated through API with ASAN (heights 1 and 2) Fixes: out of array access Fixes: m5jVlJdov0Lp Fixes: AISLE-2026-0150-00558 Found-by: Joshua Rogers (AISLE) Signed-off-by: Michael Niedermayer <[email protected]> --- libavcodec/huffyuvenc.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libavcodec/huffyuvenc.c b/libavcodec/huffyuvenc.c index 9aec2cc462f..c9fd80b838c 100644 --- a/libavcodec/huffyuvenc.c +++ b/libavcodec/huffyuvenc.c @@ -370,6 +370,13 @@ static av_cold int encode_init(AVCodecContext *avctx) return AVERROR(EINVAL); } + if (s->predictor == MEDIAN && s->version <= 2 && + (avctx->width < 4 || + AV_CEIL_RSHIFT(avctx->height, s->chroma_v_shift) < 2 + s->interlaced)) { + av_log(avctx, AV_LOG_ERROR, "Frame too small for the median predictor\n"); + return AVERROR(EINVAL); + } + avctx->extradata[0] = s->predictor | (s->decorrelate << 6); avctx->extradata[2] = s->interlaced ? 0x10 : 0x20; if (s->context) -- 2.54.0 >From 028cc498488077e9ce8c1121ada314e4087df9a6 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 24 Oct 2026 06:38:34 +0200 Subject: [PATCH 3/5] avcodec/huffyuvenc: do not left-predict a second row that a plane does not have Out of array read Replicated through API with ASAN Fixes: out of array read Fixes: TfmbebVtuO9d Fixes: AISLE-2026-0150-00921 Found-by: Joshua Rogers (AISLE) Signed-off-by: Michael Niedermayer <[email protected]> --- libavcodec/huffyuvenc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/huffyuvenc.c b/libavcodec/huffyuvenc.c index c9fd80b838c..07195576671 100644 --- a/libavcodec/huffyuvenc.c +++ b/libavcodec/huffyuvenc.c @@ -911,7 +911,7 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt, if (s->predictor==MEDIAN) { int lefttop; y = 1; - if (s->interlaced) { + if (s->interlaced && h > 1) { left = sub_left_prediction(s, s->temp[0], p->data[plane] + p->linesize[plane], w , left); encode_plane_bitstream(s, w, plane); -- 2.54.0 >From f518700c8a18a28180ee06492b4f484d9d007731 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Thu, 29 Oct 2026 02:11:48 +0100 Subject: [PATCH 4/5] avcodec/riscv/llvidencdsp: return from diff_bytes for lengths of zero or less Heap corruption Replicated through UnModified FFmpeg with qemu-riscv64 (RVV) Fixes: heap corruption Fixes: CDfb2GDVrz5J Fixes: AISLE-2026-0150-00800 Found-by: Joshua Rogers (AISLE) Signed-off-by: Michael Niedermayer <[email protected]> --- libavcodec/riscv/llvidencdsp_rvv.S | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavcodec/riscv/llvidencdsp_rvv.S b/libavcodec/riscv/llvidencdsp_rvv.S index 35ac976dd03..beedf21b758 100644 --- a/libavcodec/riscv/llvidencdsp_rvv.S +++ b/libavcodec/riscv/llvidencdsp_rvv.S @@ -22,6 +22,7 @@ func ff_llvidenc_diff_bytes_rvv, zve32x lpad 0 + blez a3, 2f 1: vsetvli t0, a3, e8, m8, ta, ma vle8.v v0, (a1) @@ -33,7 +34,7 @@ func ff_llvidenc_diff_bytes_rvv, zve32x vse8.v v8, (a0) add a0, t0, a0 bnez a3, 1b - +2: ret endfunc -- 2.54.0 >From 510b71732e528bd846e45981e057070da9c5b7f9 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 24 Oct 2026 06:38:34 +0200 Subject: [PATCH 5/5] avcodec/proresenc_anatoliy: compute the packet size bound in 64 bits Signed integer overflow Replicated through UnModified FFmpeg with UBSAN Fixes: signed integer overflow Found during triage of the security report EMbkV19WJjmA Signed-off-by: Michael Niedermayer <[email protected]> --- libavcodec/proresenc_anatoliy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/proresenc_anatoliy.c b/libavcodec/proresenc_anatoliy.c index da2deb274fe..d8c5a483004 100644 --- a/libavcodec/proresenc_anatoliy.c +++ b/libavcodec/proresenc_anatoliy.c @@ -734,7 +734,7 @@ static int prores_encode_frame(AVCodecContext *avctx, AVPacket *pkt, uint8_t *buf; int compress_frame_size, pic_size, ret, is_top_field_first = 0; uint8_t frame_flags; - int frame_size = FFALIGN(avctx->width, 16) * FFALIGN(avctx->height, 16)*16 + 500 + FF_INPUT_BUFFER_MIN_SIZE; //FIXME choose tighter limit + int64_t frame_size = (int64_t)FFALIGN(avctx->width, 16) * FFALIGN(avctx->height, 16)*16 + 500 + FF_INPUT_BUFFER_MIN_SIZE; //FIXME choose tighter limit if ((ret = ff_alloc_packet(avctx, pkt, frame_size + FF_INPUT_BUFFER_MIN_SIZE)) < 0) -- 2.54.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
