vlc | branch: master | Rémi Denis-Courmont <r...@remlab.net> | Sat Feb 10 20:57:14 2018 +0200| [50251bfbaee96d061f950dfbfc39b63797e50917] | committer: Rémi Denis-Courmont
picture: check for overflow in picture_Setup() > http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=50251bfbaee96d061f950dfbfc39b63797e50917 --- src/misc/picture.c | 48 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/src/misc/picture.c b/src/misc/picture.c index a35ff0ff15..490b4a8300 100644 --- a/src/misc/picture.c +++ b/src/misc/picture.c @@ -32,6 +32,7 @@ # include "config.h" #endif #include <assert.h> +#include <limits.h> #include <vlc_common.h> #include "picture.h" @@ -161,9 +162,10 @@ int picture_Setup( picture_t *p_picture, const video_format_t *restrict fmt ) Which is respected if you have V % lcm( p_dsc->p[0..planes].w.i_den * 16) == 0 */ - int i_modulo_w = 1; - int i_modulo_h = 1; - unsigned int i_ratio_h = 1; + unsigned i_modulo_w = 1; + unsigned i_modulo_h = 1; + unsigned i_ratio_h = 1; + for( unsigned i = 0; i < p_dsc->plane_count; i++ ) { i_modulo_w = LCM( i_modulo_w, 16 * p_dsc->p[i].w.den ); @@ -173,22 +175,44 @@ int picture_Setup( picture_t *p_picture, const video_format_t *restrict fmt ) } i_modulo_h = LCM( i_modulo_h, 32 ); - const int i_width_aligned = ( fmt->i_width + i_modulo_w - 1 ) / i_modulo_w * i_modulo_w; - const int i_height_aligned = ( fmt->i_height + i_modulo_h - 1 ) / i_modulo_h * i_modulo_h; - const int i_height_extra = 2 * i_ratio_h; /* This one is a hack for some ASM functions */ + unsigned width, height; + + if (unlikely(add_overflow(fmt->i_width, i_modulo_w - 1, &width)) + || unlikely(add_overflow(fmt->i_height, i_modulo_h - 1, &height))) + return VLC_EGENERIC; + + width = width / i_modulo_w * i_modulo_w; + height = height / i_modulo_h * i_modulo_h; + + /* Hack: append two scan lines for some SIMD assembler */ + if (unlikely(add_overflow(height, 2 * i_ratio_h, &height))) + return VLC_EGENERIC; + + /* plane_t uses 'int'. */ + if (unlikely(width > INT_MAX) || unlikely(height > INT_MAX)) + return VLC_EGENERIC; + for( unsigned i = 0; i < p_dsc->plane_count; i++ ) { plane_t *p = &p_picture->p[i]; + const vlc_rational_t *h = &p_dsc->p[i].h; + const vlc_rational_t *w = &p_dsc->p[i].w; + + /* A plane cannot be over-sampled. This could lead to overflow. */ + assert(h->den >= h->num); + assert(w->den >= w->num); + + p->i_lines = height * h->num / h->den; + p->i_visible_lines = fmt->i_visible_height * h->num / h->den; - p->i_lines = (i_height_aligned + i_height_extra ) * p_dsc->p[i].h.num / p_dsc->p[i].h.den; - p->i_visible_lines = fmt->i_visible_height * p_dsc->p[i].h.num / p_dsc->p[i].h.den; - p->i_pitch = i_width_aligned * p_dsc->p[i].w.num / p_dsc->p[i].w.den * p_dsc->pixel_size; - p->i_visible_pitch = fmt->i_visible_width * p_dsc->p[i].w.num / p_dsc->p[i].w.den * p_dsc->pixel_size; - p->i_pixel_pitch = p_dsc->pixel_size; + p->i_pitch = width * w->num / w->den * p_dsc->pixel_size; + p->i_visible_pitch = fmt->i_visible_width * w->num / w->den + * p_dsc->pixel_size; + p->i_pixel_pitch = p_dsc->pixel_size; assert( (p->i_pitch % 16) == 0 ); } - p_picture->i_planes = p_dsc->plane_count; + p_picture->i_planes = p_dsc->plane_count; return VLC_SUCCESS; } _______________________________________________ vlc-commits mailing list vlc-commits@videolan.org https://mailman.videolan.org/listinfo/vlc-commits