[libav-devel] [PATCH 10/10] pcmdec: fix output buffer size check by calculating the actual output size prior to decoding.

[Justin Ruggles]

---
 libavcodec/pcm.c |   15 +++++++++++----
 1 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/libavcodec/pcm.c b/libavcodec/pcm.c
index 8d93992..c9eb543 100644
--- a/libavcodec/pcm.c
+++ b/libavcodec/pcm.c
@@ -250,7 +250,7 @@ static int pcm_decode_frame(AVCodecContext *avctx,
     const uint8_t *src = avpkt->data;
     int buf_size = avpkt->size;
     PCMDecode *s = avctx->priv_data;
-    int sample_size, c, n;
+    int sample_size, c, n, out_size;
     uint8_t *samples;
     int32_t *dst_int32_t;
 
@@ -286,10 +286,17 @@ static int pcm_decode_frame(AVCodecContext *avctx,
             buf_size -= buf_size % n;
     }
 
-    buf_size= FFMIN(buf_size, *data_size/2);
-
     n = buf_size/sample_size;
 
+    out_size = n * av_get_bytes_per_sample(avctx->sample_fmt);
+    if (avctx->codec_id == CODEC_ID_PCM_DVD ||
+        avctx->codec_id == CODEC_ID_PCM_LXF)
+        out_size *= 2;
+    if (*data_size < out_size) {
+        av_log(avctx, AV_LOG_ERROR, "output buffer too small\n");
+        return AVERROR(EINVAL);
+    }
+
     switch(avctx->codec->id) {
     case CODEC_ID_PCM_U32LE:
         DECODE(32, le32, src, samples, n, 0, 0x80000000)
@@ -450,7 +457,7 @@ static int pcm_decode_frame(AVCodecContext *avctx,
     default:
         return -1;
     }
-    *data_size = samples - (uint8_t *)data;
+    *data_size = out_size;
     return buf_size;
 }
 
-- 
1.7.1

_______________________________________________
libav-devel mailing list
libav-devel@libav.org
https://lists.libav.org/mailman/listinfo/libav-devel