[libav-devel] [PATCH] av_picture_pad: fix handling of linesize (a.k.a. stride)

[John Stebbins]

It failed to account for line stride.  if linesize[i] != (width >> x_shift)
the padding was set incorrectly.
---
 libavcodec/imgconvert.c | 74 ++++++++++++++++++++++++++++++-------------------
 1 file changed, 45 insertions(+), 29 deletions(-)

diff --git a/libavcodec/imgconvert.c b/libavcodec/imgconvert.c
index 53e7df8..d421c37 100644
--- a/libavcodec/imgconvert.c
+++ b/libavcodec/imgconvert.c
@@ -283,9 +283,13 @@ int av_picture_pad(AVPicture *dst, const AVPicture *src, 
int height, int width,
     uint8_t *optr;
     int y_shift;
     int x_shift;
-    int yheight;
+    int src_width, src_height;
+    int top, bottom, left, right;
+    int src_h, src_w;
     int i, y;
 
+    src_width  = width   - (padleft + padright);
+    src_height = height  - (padtop  + padbottom);
     if (pix_fmt < 0 || pix_fmt >= AV_PIX_FMT_NB ||
         !is_yuv_planar(desc)) return -1;
 
@@ -293,44 +297,56 @@ int av_picture_pad(AVPicture *dst, const AVPicture *src, 
int height, int width,
         x_shift = i ? desc->log2_chroma_w : 0;
         y_shift = i ? desc->log2_chroma_h : 0;
 
-        if (padtop || padleft) {
-            memset(dst->data[i], color[i],
-                dst->linesize[i] * (padtop >> y_shift) + (padleft >> x_shift));
-        }
-
-        if (padleft || padright) {
-            optr = dst->data[i] + dst->linesize[i] * (padtop >> y_shift) +
-                (dst->linesize[i] - (padright >> x_shift));
-            yheight = (height - 1 - (padtop + padbottom)) >> y_shift;
-            for (y = 0; y < yheight; y++) {
-                memset(optr, color[i], (padleft + padright) >> x_shift);
+        top    = padtop >> y_shift;
+        bottom = padbottom >> y_shift;
+        left   = padleft >> x_shift;
+        right  = padright >> x_shift;
+        src_h  = src_height >> y_shift;
+        src_w  = src_width  >> x_shift;
+
+        // Render pad top
+        if (top) {
+            optr = dst->data[i];
+            for (y = 0; y < top; y++) {
+                memset(optr, color[i], width >> x_shift);
+                // skip stride bytes
                 optr += dst->linesize[i];
             }
         }
 
-        if (src) { /* first line */
+        // Render pad left, src image, and pad right
+        if (src) {
             uint8_t *iptr = src->data[i];
-            optr = dst->data[i] + dst->linesize[i] * (padtop >> y_shift) +
-                    (padleft >> x_shift);
-            memcpy(optr, iptr, (width - padleft - padright) >> x_shift);
-            iptr += src->linesize[i];
-            optr = dst->data[i] + dst->linesize[i] * (padtop >> y_shift) +
-                (dst->linesize[i] - (padright >> x_shift));
-            yheight = (height - 1 - (padtop + padbottom)) >> y_shift;
-            for (y = 0; y < yheight; y++) {
-                memset(optr, color[i], (padleft + padright) >> x_shift);
-                memcpy(optr + ((padleft + padright) >> x_shift), iptr,
-                       (width - padleft - padright) >> x_shift);
+            optr = dst->data[i] + dst->linesize[i] * top;
+            for (y = 0; y < src_h; y++) {
+                memset(optr, color[i], left);
+                memcpy(optr + left, iptr, src_w);
+                memset(optr + left + src_w, color[i], right);
+                // skip stride bytes
                 iptr += src->linesize[i];
                 optr += dst->linesize[i];
             }
         }
+        else if (left || right)
+        {
+            optr = dst->data[i] + dst->linesize[i] * top;
+            for (y = 0; y < src_h; y++) {
+                // padright, pad left next line
+                memset(optr, color[i], left);
+                memset(optr + left + src_w, color[i], right);
+                // skip stride bytes
+                optr += dst->linesize[i];
+            }
+        }
 
-        if (padbottom || padright) {
-            optr = dst->data[i] + dst->linesize[i] *
-                ((height - padbottom) >> y_shift) - (padright >> x_shift);
-            memset(optr, color[i],dst->linesize[i] *
-                (padbottom >> y_shift) + (padright >> x_shift));
+        // Render pad bottom
+        if (bottom) {
+            optr = dst->data[i] + dst->linesize[i] * (top + src_h);
+            for (y = 0; y < bottom; y++) {
+                memset(optr, color[i], width >> x_shift);
+                // skip stride bytes
+                optr += dst->linesize[i];
+            }
         }
     }
     return 0;
-- 
2.4.3

_______________________________________________
libav-devel mailing list
libav-devel@libav.org
https://lists.libav.org/mailman/listinfo/libav-devel