The number of bits in a PutBitsContext must fit into an int, yet the various cbs functions to write units did not check that the argument in init_put_bits is in the range 0..INT_MAX / 8. This has been changed.
Furthermore, the check 8 * data_size > data_bit_start that there is data beyond the initial padding could also overflow, so divide it by 8 to get an equivalent check without this problem. Signed-off-by: Andreas Rheinhardt <andreas.rheinha...@gmail.com> --- libavcodec/cbs_av1.c | 4 +++- libavcodec/cbs_h2645.c | 6 ++++-- libavcodec/cbs_jpeg.c | 4 +++- libavcodec/cbs_mpeg2.c | 6 ++++-- libavcodec/cbs_vp9.c | 4 +++- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/libavcodec/cbs_av1.c b/libavcodec/cbs_av1.c index cca112ecf8..f091c5ac41 100644 --- a/libavcodec/cbs_av1.c +++ b/libavcodec/cbs_av1.c @@ -1239,7 +1239,9 @@ static int cbs_av1_write_unit(CodedBitstreamContext *ctx, err = cbs_av1_write_obu(ctx, unit, &pbc); if (err == AVERROR(ENOSPC)) { // Overflow. - priv->write_buffer_size *= 2; + if (priv->write_buffer_size == INT_MAX / 8) + return AVERROR(ENOMEM); + priv->write_buffer_size = FFMIN(2 * priv->write_buffer_size, INT_MAX / 8); goto reallocate_and_try_again; } if (err < 0) diff --git a/libavcodec/cbs_h2645.c b/libavcodec/cbs_h2645.c index 5dd371153a..6025035e22 100644 --- a/libavcodec/cbs_h2645.c +++ b/libavcodec/cbs_h2645.c @@ -1101,7 +1101,7 @@ static int cbs_h2645_write_slice_data(CodedBitstreamContext *ctx, const uint8_t *pos = data + data_bit_start / 8; av_assert0(data_bit_start >= 0 && - 8 * data_size > data_bit_start); + data_size > data_bit_start / 8); if (data_size * 8 + 8 > put_bits_left(pbc)) return AVERROR(ENOSPC); @@ -1411,7 +1411,9 @@ static int cbs_h2645_write_nal_unit(CodedBitstreamContext *ctx, if (err == AVERROR(ENOSPC)) { // Overflow. - priv->write_buffer_size *= 2; + if (priv->write_buffer_size == INT_MAX / 8) + return AVERROR(ENOMEM); + priv->write_buffer_size = FFMIN(2 * priv->write_buffer_size, INT_MAX / 8); goto reallocate_and_try_again; } // Overflow but we didn't notice. diff --git a/libavcodec/cbs_jpeg.c b/libavcodec/cbs_jpeg.c index a20f062f1b..71679ef5df 100644 --- a/libavcodec/cbs_jpeg.c +++ b/libavcodec/cbs_jpeg.c @@ -406,7 +406,9 @@ static int cbs_jpeg_write_unit(CodedBitstreamContext *ctx, if (err == AVERROR(ENOSPC)) { // Overflow. - priv->write_buffer_size *= 2; + if (priv->write_buffer_size == INT_MAX / 8) + return AVERROR(ENOMEM); + priv->write_buffer_size = FFMIN(2 * priv->write_buffer_size, INT_MAX / 8); goto reallocate_and_try_again; } if (err < 0) { diff --git a/libavcodec/cbs_mpeg2.c b/libavcodec/cbs_mpeg2.c index 52252093b3..8591ada7bf 100644 --- a/libavcodec/cbs_mpeg2.c +++ b/libavcodec/cbs_mpeg2.c @@ -337,7 +337,7 @@ static int cbs_mpeg2_write_slice(CodedBitstreamContext *ctx, uint8_t *pos = slice->data + slice->data_bit_start / 8; av_assert0(slice->data_bit_start >= 0 && - 8 * slice->data_size > slice->data_bit_start); + slice->data_size > slice->data_bit_start / 8); if (slice->data_size * 8 + 8 > put_bits_left(pbc)) return AVERROR(ENOSPC); @@ -400,7 +400,9 @@ static int cbs_mpeg2_write_unit(CodedBitstreamContext *ctx, if (err == AVERROR(ENOSPC)) { // Overflow. - priv->write_buffer_size *= 2; + if (priv->write_buffer_size == INT_MAX / 8) + return AVERROR(ENOMEM); + priv->write_buffer_size = FFMIN(2 * priv->write_buffer_size, INT_MAX / 8); goto reallocate_and_try_again; } if (err < 0) { diff --git a/libavcodec/cbs_vp9.c b/libavcodec/cbs_vp9.c index ebb4c573ea..88191fee3d 100644 --- a/libavcodec/cbs_vp9.c +++ b/libavcodec/cbs_vp9.c @@ -548,7 +548,9 @@ static int cbs_vp9_write_unit(CodedBitstreamContext *ctx, err = cbs_vp9_write_frame(ctx, &pbc, frame); if (err == AVERROR(ENOSPC)) { - priv->write_buffer_size *= 2; + if (priv->write_buffer_size == INT_MAX / 8) + return AVERROR(ENOMEM); + priv->write_buffer_size = FFMIN(2 * priv->write_buffer_size, INT_MAX / 8); goto reallocate_and_try_again; } if (err < 0) -- 2.20.1 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".