Module: libav Branch: release/0.8 Commit: c02b9e6e633896617e5f95211665c5521800498b
Author: Luca Barbato <lu_z...@gentoo.org> Committer: Luca Barbato <lu_z...@gentoo.org> Date: Fri Jul 12 14:33:24 2013 +0200 indeo: Bound-check before applying transform Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-sta...@libav.org (cherry picked from commit dc79685195a45c9b8b17d7b93d118e0aefa45462) Signed-off-by: Luca Barbato <lu_z...@gentoo.org> Conflicts: libavcodec/ivi_common.c --- libavcodec/indeo4.c | 7 +++++++ libavcodec/indeo5.c | 38 +++++++++++++++++++++++--------------- libavcodec/ivi_common.c | 31 ++++++++++++++++++++++++++++--- libavcodec/ivi_common.h | 1 + 4 files changed, 59 insertions(+), 18 deletions(-) diff --git a/libavcodec/indeo4.c b/libavcodec/indeo4.c index a3b3c6b..74b3ef0 100644 --- a/libavcodec/indeo4.c +++ b/libavcodec/indeo4.c @@ -348,6 +348,13 @@ static int decode_band_hdr(IVI45DecContext *ctx, IVIBandDesc *band, band->inv_transform = transforms[transform_id].inv_trans; band->dc_transform = transforms[transform_id].dc_trans; band->is_2d_trans = transforms[transform_id].is_2d_trans; + if (transform_id < 10) + band->transform_size = 8; + else + band->transform_size = 4; + + if (band->blk_size != band->transform_size) + return AVERROR_INVALIDDATA; scan_indx = get_bits(&ctx->gb, 4); if (scan_indx == 15) { diff --git a/libavcodec/indeo5.c b/libavcodec/indeo5.c index 0626454..c06d46d 100644 --- a/libavcodec/indeo5.c +++ b/libavcodec/indeo5.c @@ -147,39 +147,47 @@ static int decode_gop_header(IVI45DecContext *ctx, AVCodecContext *avctx) /* select transform function and scan pattern according to plane and band number */ switch ((p << 2) + i) { case 0: - band->inv_transform = ff_ivi_inverse_slant_8x8; - band->dc_transform = ff_ivi_dc_slant_2d; - band->scan = ff_zigzag_direct; + band->inv_transform = ff_ivi_inverse_slant_8x8; + band->dc_transform = ff_ivi_dc_slant_2d; + band->scan = ff_zigzag_direct; + band->transform_size = 8; break; case 1: - band->inv_transform = ff_ivi_row_slant8; - band->dc_transform = ff_ivi_dc_row_slant; - band->scan = ff_ivi_vertical_scan_8x8; + band->inv_transform = ff_ivi_row_slant8; + band->dc_transform = ff_ivi_dc_row_slant; + band->scan = ff_ivi_vertical_scan_8x8; + band->transform_size = 8; break; case 2: - band->inv_transform = ff_ivi_col_slant8; - band->dc_transform = ff_ivi_dc_col_slant; - band->scan = ff_ivi_horizontal_scan_8x8; + band->inv_transform = ff_ivi_col_slant8; + band->dc_transform = ff_ivi_dc_col_slant; + band->scan = ff_ivi_horizontal_scan_8x8; + band->transform_size = 8; break; case 3: - band->inv_transform = ff_ivi_put_pixels_8x8; - band->dc_transform = ff_ivi_put_dc_pixel_8x8; - band->scan = ff_ivi_horizontal_scan_8x8; + band->inv_transform = ff_ivi_put_pixels_8x8; + band->dc_transform = ff_ivi_put_dc_pixel_8x8; + band->scan = ff_ivi_horizontal_scan_8x8; + band->transform_size = 8; break; case 4: - band->inv_transform = ff_ivi_inverse_slant_4x4; - band->dc_transform = ff_ivi_dc_slant_2d; - band->scan = ff_ivi_direct_scan_4x4; + band->inv_transform = ff_ivi_inverse_slant_4x4; + band->dc_transform = ff_ivi_dc_slant_2d; + band->scan = ff_ivi_direct_scan_4x4; + band->transform_size = 4; break; } band->is_2d_trans = band->inv_transform == ff_ivi_inverse_slant_8x8 || band->inv_transform == ff_ivi_inverse_slant_4x4; + if (band->transform_size != band->blk_size) + return AVERROR_INVALIDDATA; + /* select dequant matrix according to plane and band number */ if (!p) { quant_mat = (pic_conf.luma_bands > 1) ? i+1 : 0; diff --git a/libavcodec/ivi_common.c b/libavcodec/ivi_common.c index 4aab7ab..ea9082c 100644 --- a/libavcodec/ivi_common.c +++ b/libavcodec/ivi_common.c @@ -365,6 +365,25 @@ int ff_ivi_dec_tile_data_size(GetBitContext *gb) return len; } +static int ivi_dc_transform(IVIBandDesc *band, int *prev_dc, int buf_offs, + int blk_size) +{ + int buf_size = band->pitch * band->aheight - buf_offs; + int min_size = (blk_size - 1) * band->pitch + blk_size; + + if (!band->dc_transform) + return 0; + + + if (min_size > buf_size) + return AVERROR_INVALIDDATA; + + band->dc_transform(prev_dc, band->buf + buf_offs, + band->pitch, blk_size); + + return 0; +} + static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band, ivi_mc_func mc, int mv_x, int mv_y, int *prev_dc, int is_intra, int mc_type, @@ -380,6 +399,12 @@ static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band, int num_coeffs = blk_size * blk_size; int col_mask = blk_size - 1; int scan_pos = -1; + int min_size = band->pitch * (band->transform_size - 1) + + band->transform_size; + int buf_size = band->pitch * band->aheight - offs; + + if (min_size > buf_size) + return AVERROR_INVALIDDATA; if (!band->scan) return AVERROR_INVALIDDATA; @@ -526,9 +551,9 @@ int ff_ivi_decode_blocks(GetBitContext *gb, IVIBandDesc *band, IVITile *tile) /* for intra blocks apply the dc slant transform */ /* for inter - perform the motion compensation without delta */ if (is_intra) { - if (band->dc_transform) - band->dc_transform(&prev_dc, band->buf + buf_offs, - band->pitch, blk_size); + ret = ivi_dc_transform(band, &prev_dc, buf_offs, blk_size); + if (ret < 0) + return ret; } else { ret = ivi_mc(mc_no_delta_func, band->buf, band->ref_buf, buf_offs, mv_x, mv_y, band->pitch, mc_type); diff --git a/libavcodec/ivi_common.h b/libavcodec/ivi_common.h index 07736f2..b47d5d2 100644 --- a/libavcodec/ivi_common.h +++ b/libavcodec/ivi_common.h @@ -162,6 +162,7 @@ typedef struct { int num_tiles; ///< number of tiles in this band IVITile *tiles; ///< array of tile descriptors InvTransformPtr *inv_transform; + int transform_size; DCTransformPtr *dc_transform; int is_2d_trans; ///< 1 indicates that the two-dimensional inverse transform is used int32_t checksum; ///< for debug purposes _______________________________________________ libav-commits mailing list libav-commits@libav.org https://lists.libav.org/mailman/listinfo/libav-commits