[libav-devel] [PATCH] dnxhdenc: fix invalid reads in dnxhd_mb_var_thread().

2013-01-22 Thread Anton Khirnov
Do not assume that frame dimensions are mod16 (or that height is mod32
for interlaced).
---
 libavcodec/dnxhdenc.c|   27 ---
 tests/ref/vsynth/vsynth1-dnxhd-1080i |4 ++--
 tests/ref/vsynth/vsynth2-dnxhd-1080i |4 ++--
 3 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/libavcodec/dnxhdenc.c b/libavcodec/dnxhdenc.c
index 8531fe0..97e0fed 100644
--- a/libavcodec/dnxhdenc.c
+++ b/libavcodec/dnxhdenc.c
@@ -615,14 +615,35 @@ static void dnxhd_setup_threads_slices(DNXHDEncContext 
*ctx)
 static int dnxhd_mb_var_thread(AVCodecContext *avctx, void *arg, int jobnr, 
int threadnr)
 {
 DNXHDEncContext *ctx = avctx-priv_data;
-int mb_y = jobnr, mb_x;
+int mb_y = jobnr, mb_x, x, y;
+int partial_last_row = (mb_y == ctx-m.mb_height - 1) 
+   ((avctx-height  ctx-interlaced)  0xF);
+
 ctx = ctx-thread[threadnr];
 if (ctx-cid_table-bit_depth == 8) {
 uint8_t *pix = ctx-thread[0]-src[0] + ((mb_y4) * ctx-m.linesize);
 for (mb_x = 0; mb_x  ctx-m.mb_width; ++mb_x, pix += 16) {
 unsigned mb  = mb_y * ctx-m.mb_width + mb_x;
-int sum = ctx-m.dsp.pix_sum(pix, ctx-m.linesize);
-int varc = (ctx-m.dsp.pix_norm1(pix, ctx-m.linesize) - 
(((unsigned)sum*sum)8)+128)8;
+int sum;
+int varc;
+
+if (!partial_last_row  mb_x * 16 = avctx-width - 16) {
+sum  = ctx-m.dsp.pix_sum(pix, ctx-m.linesize);
+varc = ctx-m.dsp.pix_norm1(pix, ctx-m.linesize);
+} else {
+int bw = FFMIN(avctx-width - 16 * mb_x, 16);
+int bh = FFMIN((avctx-height  ctx-interlaced) - 16 * mb_y, 
16);
+sum = varc = 0;
+for (y = 0; y  bh; y++) {
+for (x = 0; x  bw; x++) {
+uint8_t val = pix[x + y * ctx-m.linesize];
+sum  += val;
+varc += val * val;
+}
+}
+}
+varc = (varc - (((unsigned)sum * sum)  8) + 128)  8;
+
 ctx-mb_cmp[mb].value = varc;
 ctx-mb_cmp[mb].mb = mb;
 }
diff --git a/tests/ref/vsynth/vsynth1-dnxhd-1080i 
b/tests/ref/vsynth/vsynth1-dnxhd-1080i
index 1eddbf8..3a990c5 100644
--- a/tests/ref/vsynth/vsynth1-dnxhd-1080i
+++ b/tests/ref/vsynth/vsynth1-dnxhd-1080i
@@ -1,4 +1,4 @@
-3cfbe36a7dd5b48859b8a569d626ef77 *tests/data/fate/vsynth1-dnxhd-1080i.mov
+2412f206f5efcbbcc3f2bba0c86b73d4 *tests/data/fate/vsynth1-dnxhd-1080i.mov
 3031875 tests/data/fate/vsynth1-dnxhd-1080i.mov
-0c651e840f860592f0d5b66030d9fa32 
*tests/data/fate/vsynth1-dnxhd-1080i.out.rawvideo
+34076f61254997c8157eafed1c916472 
*tests/data/fate/vsynth1-dnxhd-1080i.out.rawvideo
 stddev:6.29 PSNR: 32.15 MAXDIFF:   64 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth2-dnxhd-1080i 
b/tests/ref/vsynth/vsynth2-dnxhd-1080i
index 41a8d51..27c79a5 100644
--- a/tests/ref/vsynth/vsynth2-dnxhd-1080i
+++ b/tests/ref/vsynth/vsynth2-dnxhd-1080i
@@ -1,4 +1,4 @@
-19a91b7da35cecf41e5e3cb322485627 *tests/data/fate/vsynth2-dnxhd-1080i.mov
+65ca6385b565b6ea9a2e28150eef1d46 *tests/data/fate/vsynth2-dnxhd-1080i.mov
 3031875 tests/data/fate/vsynth2-dnxhd-1080i.mov
-3c559af629ae0a8fb1a9a0e4b4da7733 
*tests/data/fate/vsynth2-dnxhd-1080i.out.rawvideo
+42262a2325441b38b3b3c8a42d888e7d 
*tests/data/fate/vsynth2-dnxhd-1080i.out.rawvideo
 stddev:1.31 PSNR: 45.77 MAXDIFF:   23 bytes:  7603200/   760320
-- 
1.7.10.4

___
libav-devel mailing list
libav-devel@libav.org
https://lists.libav.org/mailman/listinfo/libav-devel


Re: [libav-devel] [PATCH] dnxhdenc: fix invalid reads in dnxhd_mb_var_thread().

2013-01-22 Thread Kostya Shishkov
On Tue, Jan 22, 2013 at 12:04:56PM +0100, Anton Khirnov wrote:
 Do not assume that frame dimensions are mod16 (or that height is mod32
 for interlaced).
 ---
  libavcodec/dnxhdenc.c|   27 ---
  tests/ref/vsynth/vsynth1-dnxhd-1080i |4 ++--
  tests/ref/vsynth/vsynth2-dnxhd-1080i |4 ++--
  3 files changed, 28 insertions(+), 7 deletions(-)

probably OK but I suspect Diego needs to review this as well
___
libav-devel mailing list
libav-devel@libav.org
https://lists.libav.org/mailman/listinfo/libav-devel