---
This takes care of all remaining *alloc* checks in libavcodec.
It's grouped in a single patch since it's nothing more than
if (!var) return AVERROR(ENOMEM);
for the most part (no propagation or return type changes).
These are all spotted thanks to Coverity and a proper model file.
Cheers,
Vittorio
libavcodec/aacpsy.c | 6 ++++++
libavcodec/asvenc.c | 2 ++
libavcodec/dct.c | 2 ++
libavcodec/eatgv.c | 2 ++
libavcodec/ffv1.c | 2 ++
libavcodec/flacenc.c | 2 ++
libavcodec/huffyuvenc.c | 2 +-
libavcodec/jpeglsdec.c | 8 ++++++++
libavcodec/jpeglsenc.c | 6 ++++++
libavcodec/lclenc.c | 2 ++
libavcodec/libtheoraenc.c | 6 ++++++
libavcodec/libx264.c | 4 ++++
libavcodec/libxvid.c | 6 ++++++
libavcodec/psymodel.c | 12 ++++++++++++
libavcodec/pthread_frame.c | 5 +++++
libavcodec/ratecontrol.c | 5 +++++
libavcodec/svq1enc.c | 8 ++++++++
libavcodec/truemotion2.c | 9 +++++++++
libavcodec/wmaenc.c | 4 ++++
libavcodec/xsubdec.c | 13 +++++++++++++
20 files changed, 105 insertions(+), 1 deletion(-)
diff --git a/libavcodec/aacpsy.c b/libavcodec/aacpsy.c
index 6c6e573..15f872e 100644
--- a/libavcodec/aacpsy.c
+++ b/libavcodec/aacpsy.c
@@ -298,6 +298,8 @@ static av_cold int psy_3gpp_init(FFPsyContext *ctx) {
const float num_bark = calc_bark((float)bandwidth);
ctx->model_priv_data = av_mallocz(sizeof(AacPsyContext));
+ if (!ctx->model_priv_data)
+ return AVERROR(ENOMEM);
pctx = (AacPsyContext*) ctx->model_priv_data;
pctx->chan_bitrate = chan_bitrate;
@@ -349,6 +351,10 @@ static av_cold int psy_3gpp_init(FFPsyContext *ctx) {
}
pctx->ch = av_mallocz(sizeof(AacPsyChannel) * ctx->avctx->channels);
+ if (!pctx->ch) {
+ av_freep(&pctx);
+ return AVERROR(ENOMEM);
+ }
lame_window_init(pctx, ctx->avctx);
diff --git a/libavcodec/asvenc.c b/libavcodec/asvenc.c
index d865c2e..d016eac 100644
--- a/libavcodec/asvenc.c
+++ b/libavcodec/asvenc.c
@@ -298,6 +298,8 @@ static av_cold int encode_init(AVCodecContext *avctx)
avctx->global_quality / 2) / avctx->global_quality;
avctx->extradata = av_mallocz(8);
+ if (!avctx->extradata)
+ return AVERROR(ENOMEM);
avctx->extradata_size = 8;
((uint32_t *) avctx->extradata)[0] = av_le2ne32(a->inv_qscale);
((uint32_t *) avctx->extradata)[1] = av_le2ne32(AV_RL32("ASUS"));
diff --git a/libavcodec/dct.c b/libavcodec/dct.c
index 4dbbff8..180477e 100644
--- a/libavcodec/dct.c
+++ b/libavcodec/dct.c
@@ -191,6 +191,8 @@ av_cold int ff_dct_init(DCTContext *s, int nbits, enum
DCTTransformType inverse)
s->costab = ff_cos_tabs[nbits + 2];
s->csc2 = av_malloc(n / 2 * sizeof(FFTSample));
+ if (!s->csc2)
+ return AVERROR(ENOMEM);
if (ff_rdft_init(&s->rdft, nbits, inverse == DCT_III) < 0) {
av_free(s->csc2);
diff --git a/libavcodec/eatgv.c b/libavcodec/eatgv.c
index c400b56..5df4301 100644
--- a/libavcodec/eatgv.c
+++ b/libavcodec/eatgv.c
@@ -174,6 +174,8 @@ static int tgv_decode_inter(TgvContext *s, AVFrame *frame,
/* allocate codebook buffers as necessary */
if (num_mvs > s->num_mvs) {
s->mv_codebook = av_realloc(s->mv_codebook, num_mvs*2*sizeof(int));
+ if (!s->mv_codebook)
+ return AVERROR(ENOMEM);
s->num_mvs = num_mvs;
}
diff --git a/libavcodec/ffv1.c b/libavcodec/ffv1.c
index d1a6a83..96f170f 100644
--- a/libavcodec/ffv1.c
+++ b/libavcodec/ffv1.c
@@ -203,6 +203,8 @@ av_cold int ffv1_init_slice_contexts(FFV1Context *f)
int sxe = f->avctx->width * (sx + 1) / f->num_h_slices;
int sys = f->avctx->height * sy / f->num_v_slices;
int sye = f->avctx->height * (sy + 1) / f->num_v_slices;
+ if (!fs)
+ return AVERROR(ENOMEM);
f->slice_context[i] = fs;
memcpy(fs, f, sizeof(*fs));
memset(fs->rc_stat2, 0, sizeof(fs->rc_stat2));
diff --git a/libavcodec/flacenc.c b/libavcodec/flacenc.c
index 2277cf3..5c39d84 100644
--- a/libavcodec/flacenc.c
+++ b/libavcodec/flacenc.c
@@ -626,6 +626,8 @@ static uint64_t calc_rice_params(RiceContext *rc, int pmin,
int pmax,
tmp_rc.coding_mode = rc->coding_mode;
udata = av_malloc(n * sizeof(uint32_t));
+ if (!udata)
+ return AVERROR(ENOMEM);
for (i = 0; i < n; i++)
udata[i] = (2*data[i]) ^ (data[i]>>31);
diff --git a/libavcodec/huffyuvenc.c b/libavcodec/huffyuvenc.c
index 47fe2a5..06126c8 100644
--- a/libavcodec/huffyuvenc.c
+++ b/libavcodec/huffyuvenc.c
@@ -154,7 +154,7 @@ static av_cold int encode_init(AVCodecContext *avctx)
s->version = 2;
avctx->coded_frame = av_frame_alloc();
- if (!avctx->coded_frame)
+ if (!avctx->extradata || !avctx->stats_out || !avctx->coded_frame)
return AVERROR(ENOMEM);
avctx->coded_frame->pict_type = AV_PICTURE_TYPE_I;
diff --git a/libavcodec/jpeglsdec.c b/libavcodec/jpeglsdec.c
index df72ca3..363806e 100644
--- a/libavcodec/jpeglsdec.c
+++ b/libavcodec/jpeglsdec.c
@@ -277,10 +277,18 @@ int ff_jpegls_decode_picture(MJpegDecodeContext *s, int
near,
int off = 0, stride = 1, width, shift, ret = 0;
zero = av_mallocz(s->picture_ptr->linesize[0]);
+ if (!zero) {
+ ret = AVERROR(ENOMEM);
+ goto end;
+ }
last = zero;
cur = s->picture_ptr->data[0];
state = av_mallocz(sizeof(JLSState));
+ if (!state) {
+ ret = AVERROR(ENOMEM);
+ goto end;
+ }
/* initialize JPEG-LS state from JPEG parameters */
state->near = near;
state->bpp = (s->bits < 2) ? 2 : s->bits;
diff --git a/libavcodec/jpeglsenc.c b/libavcodec/jpeglsenc.c
index 3af6412..fd2970a 100644
--- a/libavcodec/jpeglsenc.c
+++ b/libavcodec/jpeglsenc.c
@@ -270,6 +270,8 @@ static int encode_picture_ls(AVCodecContext *avctx,
AVPacket *pkt,
}
buf2 = av_malloc(pkt->size);
+ if (!buf2)
+ return AVERROR(ENOMEM);
init_put_bits(&pb, pkt->data, pkt->size);
init_put_bits(&pb2, buf2, pkt->size);
@@ -300,6 +302,8 @@ static int encode_picture_ls(AVCodecContext *avctx,
AVPacket *pkt,
put_bits(&pb, 8, 0); // point transform: none
state = av_mallocz(sizeof(JLSState));
+ if (!state)
+ return AVERROR(ENOMEM);
/* initialize JPEG-LS state from JPEG parameters */
state->near = near;
state->bpp = (avctx->pix_fmt == AV_PIX_FMT_GRAY16) ? 16 : 8;
@@ -309,6 +313,8 @@ static int encode_picture_ls(AVCodecContext *avctx,
AVPacket *pkt,
ls_store_lse(state, &pb);
zero = av_mallocz(p->linesize[0]);
+ if (!zero)
+ return AVERROR(ENOMEM);
last = zero;
cur = p->data[0];
if (avctx->pix_fmt == AV_PIX_FMT_GRAY8) {
diff --git a/libavcodec/lclenc.c b/libavcodec/lclenc.c
index acf5e73..4d2f253 100644
--- a/libavcodec/lclenc.c
+++ b/libavcodec/lclenc.c
@@ -134,6 +134,8 @@ static av_cold int encode_init(AVCodecContext *avctx)
assert(avctx->width && avctx->height);
avctx->extradata= av_mallocz(8);
+ if (!avctx->extradata)
+ return AVERROR(ENOMEM);
avctx->coded_frame = av_frame_alloc();
if (!avctx->coded_frame)
diff --git a/libavcodec/libtheoraenc.c b/libavcodec/libtheoraenc.c
index 75b0a16..f88faf4 100644
--- a/libavcodec/libtheoraenc.c
+++ b/libavcodec/libtheoraenc.c
@@ -101,6 +101,8 @@ static int get_stats(AVCodecContext *avctx, int eos)
if (!eos) {
h->stats = av_fast_realloc(h->stats, &h->stats_size,
h->stats_offset + bytes);
+ if (!h->stats)
+ return AVERROR(ENOMEM);
memcpy(h->stats + h->stats_offset, buf, bytes);
h->stats_offset += bytes;
} else {
@@ -108,6 +110,8 @@ static int get_stats(AVCodecContext *avctx, int eos)
// libtheora generates a summary header at the end
memcpy(h->stats, buf, bytes);
avctx->stats_out = av_malloc(b64_size);
+ if (!avctx->stats_out)
+ return AVERROR(ENOMEM);
av_base64_encode(avctx->stats_out, b64_size, h->stats,
h->stats_offset);
}
return 0;
@@ -131,6 +135,8 @@ static int submit_stats(AVCodecContext *avctx)
}
h->stats_size = strlen(avctx->stats_in) * 3/4;
h->stats = av_malloc(h->stats_size);
+ if (!h->stats)
+ return AVERROR(ENOMEM);
h->stats_size = av_base64_decode(h->stats, avctx->stats_in,
h->stats_size);
}
while (h->stats_size - h->stats_offset > 0) {
diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c
index 6388b6c..397cee7 100644
--- a/libavcodec/libx264.c
+++ b/libavcodec/libx264.c
@@ -531,6 +531,8 @@ static av_cold int X264_init(AVCodecContext *avctx)
s = x264_encoder_headers(x4->enc, &nal, &nnal);
avctx->extradata = p = av_malloc(s);
+ if (!p)
+ return AVERROR(ENOMEM);
for (i = 0; i < nnal; i++) {
/* Don't put the SEI in extradata. */
@@ -538,6 +540,8 @@ static av_cold int X264_init(AVCodecContext *avctx)
av_log(avctx, AV_LOG_INFO, "%s\n", nal[i].p_payload+25);
x4->sei_size = nal[i].i_payload;
x4->sei = av_malloc(x4->sei_size);
+ if (!x4->sei)
+ return AVERROR(ENOMEM);
memcpy(x4->sei, nal[i].p_payload, nal[i].i_payload);
continue;
}
diff --git a/libavcodec/libxvid.c b/libavcodec/libxvid.c
index 6486ad1..938074d 100644
--- a/libavcodec/libxvid.c
+++ b/libavcodec/libxvid.c
@@ -275,6 +275,8 @@ static int xvid_strip_vol_header(AVCodecContext *avctx,
AVPacket *pkt,
/* We need to store the header, so extract it */
if (!avctx->extradata) {
avctx->extradata = av_malloc(vo_len);
+ if (!avctx->extradata)
+ return AVERROR(ENOMEM);
memcpy(avctx->extradata, pkt->data, vo_len);
avctx->extradata_size = vo_len;
}
@@ -593,11 +595,15 @@ static av_cold int xvid_encode_init(AVCodecContext *avctx)
if (avctx->intra_matrix) {
intra = avctx->intra_matrix;
x->intra_matrix = av_malloc(sizeof(unsigned char) * 64);
+ if (!x->intra_matrix)
+ return AVERROR(ENOMEM);
} else
intra = NULL;
if (avctx->inter_matrix) {
inter = avctx->inter_matrix;
x->inter_matrix = av_malloc(sizeof(unsigned char) * 64);
+ if (!x->inter_matrix)
+ return AVERROR(ENOMEM);
} else
inter = NULL;
diff --git a/libavcodec/psymodel.c b/libavcodec/psymodel.c
index a2af611..790fa57 100644
--- a/libavcodec/psymodel.c
+++ b/libavcodec/psymodel.c
@@ -39,6 +39,12 @@ av_cold int ff_psy_init(FFPsyContext *ctx, AVCodecContext
*avctx, int num_lens,
ctx->group = av_mallocz(sizeof(ctx->group[0]) * num_groups);
ctx->bands = av_malloc (sizeof(ctx->bands[0]) * num_lens);
ctx->num_bands = av_malloc (sizeof(ctx->num_bands[0]) * num_lens);
+
+ if (!ctx->ch || !ctx->group || !ctx->bands || !ctx->num_bands) {
+ ff_psy_end(ctx);
+ return AVERROR(ENOMEM);
+ }
+
memcpy(ctx->bands, bands, sizeof(ctx->bands[0]) * num_lens);
memcpy(ctx->num_bands, num_bands, sizeof(ctx->num_bands[0]) * num_lens);
@@ -98,6 +104,8 @@ av_cold struct FFPsyPreprocessContext*
ff_psy_preprocess_init(AVCodecContext *av
int i;
float cutoff_coeff = 0;
ctx = av_mallocz(sizeof(FFPsyPreprocessContext));
+ if (!ctx)
+ return NULL;
ctx->avctx = avctx;
if (avctx->cutoff > 0)
@@ -109,6 +117,10 @@ av_cold struct FFPsyPreprocessContext*
ff_psy_preprocess_init(AVCodecContext *av
cutoff_coeff, 0.0, 0.0);
if (ctx->fcoeffs) {
ctx->fstate = av_mallocz(sizeof(ctx->fstate[0]) * avctx->channels);
+ if (!ctx->fstate) {
+ av_freep(&ctx);
+ return NULL;
+ }
for (i = 0; i < avctx->channels; i++)
ctx->fstate[i] = ff_iir_filter_init_state(FILT_ORDER);
}
diff --git a/libavcodec/pthread_frame.c b/libavcodec/pthread_frame.c
index effc9a5..e8d16a8 100644
--- a/libavcodec/pthread_frame.c
+++ b/libavcodec/pthread_frame.c
@@ -584,8 +584,13 @@ int ff_frame_thread_init(AVCodecContext *avctx)
}
avctx->internal->thread_ctx = fctx =
av_mallocz(sizeof(FrameThreadContext));
+ if (!fctx)
+ return AVERROR(ENOMEM);
fctx->threads = av_mallocz(sizeof(PerThreadContext) * thread_count);
+ if (!fctx->threads)
+ return AVERROR(ENOMEM);
+
pthread_mutex_init(&fctx->buffer_mutex, NULL);
fctx->delaying = 1;
diff --git a/libavcodec/ratecontrol.c b/libavcodec/ratecontrol.c
index e18878d..f561eb8 100644
--- a/libavcodec/ratecontrol.c
+++ b/libavcodec/ratecontrol.c
@@ -931,6 +931,11 @@ static int init_pass2(MpegEncContext *s)
qscale = av_malloc(sizeof(double) * rcc->num_entries);
blurred_qscale = av_malloc(sizeof(double) * rcc->num_entries);
+ if (!qscale || !blurred_qscale) {
+ av_freep(&qscale);
+ av_freep(&blurred_qscale);
+ return AVERROR(ENOMEM);
+ }
toobig = 0;
for (step = 256 * 256; step > 0.0000001; step *= 0.5) {
diff --git a/libavcodec/svq1enc.c b/libavcodec/svq1enc.c
index f49f487..7620332 100644
--- a/libavcodec/svq1enc.c
+++ b/libavcodec/svq1enc.c
@@ -293,6 +293,8 @@ static int svq1_encode_plane(SVQ1EncContext *s, int plane,
s->motion_val16[plane] = av_mallocz((s->m.mb_stride *
(block_height + 2) + 1) *
2 * sizeof(int16_t));
+ if (!s->motion_val8[plane] || !s->motion_val16[plane])
+ return AVERROR(ENOMEM);
}
s->m.mb_type = s->mb_type;
@@ -550,6 +552,12 @@ static av_cold int svq1_encode_init(AVCodecContext *avctx)
s->y_block_height * sizeof(int32_t));
s->ssd_int8_vs_int16 = ssd_int8_vs_int16_c;
+ if (!s->m.me.temp || !s->m.me.scratchpad || !s->m.me.map ||
+ !s->m.me.score_map || !s->mb_type || !s->dummy) {
+ svq1_encode_end(avctx);
+ return AVERROR(ENOMEM);
+ }
+
if (ARCH_PPC)
ff_svq1enc_init_ppc(s);
if (ARCH_X86)
diff --git a/libavcodec/truemotion2.c b/libavcodec/truemotion2.c
index 122643d..b56129c 100644
--- a/libavcodec/truemotion2.c
+++ b/libavcodec/truemotion2.c
@@ -171,6 +171,10 @@ static int tm2_build_huff_table(TM2Context *ctx, TM2Codes
*code)
huff.nums = av_mallocz(huff.max_num * sizeof(int));
huff.bits = av_mallocz(huff.max_num * sizeof(uint32_t));
huff.lens = av_mallocz(huff.max_num * sizeof(int));
+ if (!huff.nums || !huff.bits || !huff.lens) {
+ res = AVERROR(ENOMEM);
+ goto out;
+ }
res = tm2_read_tree(ctx, 0, 0, &huff);
@@ -193,10 +197,15 @@ static int tm2_build_huff_table(TM2Context *ctx, TM2Codes
*code)
code->bits = huff.max_bits;
code->length = huff.max_num;
code->recode = av_malloc(code->length * sizeof(int));
+ if (!code->recode) {
+ res = AVERROR(ENOMEM);
+ goto out;
+ }
for (i = 0; i < code->length; i++)
code->recode[i] = huff.nums[i];
}
}
+out:
/* free allocated memory */
av_free(huff.nums);
av_free(huff.bits);
diff --git a/libavcodec/wmaenc.c b/libavcodec/wmaenc.c
index e801663..c176daa 100644
--- a/libavcodec/wmaenc.c
+++ b/libavcodec/wmaenc.c
@@ -62,11 +62,15 @@ static av_cold int encode_init(AVCodecContext *avctx)
flags2 = 1;
if (avctx->codec->id == AV_CODEC_ID_WMAV1) {
extradata = av_malloc(4);
+ if (!extradata)
+ return AVERROR(ENOMEM);
avctx->extradata_size = 4;
AV_WL16(extradata, flags1);
AV_WL16(extradata + 2, flags2);
} else if (avctx->codec->id == AV_CODEC_ID_WMAV2) {
extradata = av_mallocz(10);
+ if (!extradata)
+ return AVERROR(ENOMEM);
avctx->extradata_size = 10;
AV_WL32(extradata, flags1);
AV_WL16(extradata + 4, flags2);
diff --git a/libavcodec/xsubdec.c b/libavcodec/xsubdec.c
index d01b410..700cfd0 100644
--- a/libavcodec/xsubdec.c
+++ b/libavcodec/xsubdec.c
@@ -95,7 +95,13 @@ static int decode_frame(AVCodecContext *avctx, void *data,
int *data_size,
// allocate sub and set values
sub->rects = av_mallocz(sizeof(*sub->rects));
+ if (!sub->rects)
+ return AVERROR(ENOMEM);
sub->rects[0] = av_mallocz(sizeof(*sub->rects[0]));
+ if (!sub->rects[0]) {
+ av_freep(&sub->rects);
+ return AVERROR(ENOMEM);
+ }
sub->num_rects = 1;
sub->rects[0]->x = x; sub->rects[0]->y = y;
sub->rects[0]->w = w; sub->rects[0]->h = h;
@@ -104,6 +110,13 @@ static int decode_frame(AVCodecContext *avctx, void *data,
int *data_size,
sub->rects[0]->pict.data[0] = av_malloc(w * h);
sub->rects[0]->nb_colors = 4;
sub->rects[0]->pict.data[1] = av_mallocz(AVPALETTE_SIZE);
+ if (!sub->rects[0]->pict.data[0] || sub->rects[0]->pict.data[1]) {
+ av_freep(&sub->rects[0]->pict.data[1]);
+ av_freep(&sub->rects[0]->pict.data[0]);
+ av_freep(&sub->rects[0]);
+ av_freep(&sub->rects);
+ return AVERROR(ENOMEM);
+ }
// read palette
for (i = 0; i < sub->rects[0]->nb_colors; i++)
--
1.9.3 (Apple Git-50)
_______________________________________________
libav-devel mailing list
[email protected]
https://lists.libav.org/mailman/listinfo/libav-devel
