>From 1b6400c1ad16635dc86de254d101094a8673a28d Mon Sep 17 00:00:00 2001
From: Uoti Urpala <uau@glyph.nonexistent.invalid>
Date: Tue, 7 Aug 2012 14:48:06 +0300
Subject: [PATCH] h264: vdpau: fix crash with unsupported colorspace
The h264_vdpau decoder crashed if output colorspace was not 8-bit 420.
Add a check to error out instead (current hardware does not support
other colorspaces, so successful decoding is not possible).
---
libavcodec/h264.c | 9 +++++++++
1 files changed, 9 insertions(+), 0 deletions(-)
diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 2d574d3..dba8649 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -3883,6 +3883,15 @@ again:
if (avctx->bits_per_raw_sample != h->sps.bit_depth_luma ||
h->cur_chroma_format_idc != h->sps.chroma_format_idc) {
+ if (s->avctx->codec->capabilities & CODEC_CAP_HWACCEL_VDPAU
+ && (h->sps.bit_depth_luma != 8 ||
+ h->sps.chroma_format_idc > 1)) {
+ av_log(avctx, AV_LOG_ERROR,
+ "VDPAU decoding does not support video "
+ "colorspace\n");
+ buf_index = -1;
+ goto end;
+ }
if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 10) {
avctx->bits_per_raw_sample = h->sps.bit_depth_luma;
h->cur_chroma_format_idc = h->sps.chroma_format_idc;
--
1.7.6.561.g3822
_______________________________________________
libav-devel mailing list
libav-devel@libav.org
https://lists.libav.org/mailman/listinfo/libav-devel
