Re: [libav-devel] [PATCH 2/2] swscale: support gray to 9bit and 10bit formats

[Luca Barbato]

On 10/26/2012 09:47 PM, Ronald S. Bultje wrote:
> Hi,
> 
> On Fri, Oct 26, 2012 at 8:35 AM, Luca Barbato  wrote:
>> Signed-off-by: Luca Barbato 
>> ---
>>
>> Partially fixes the issue, there is still something wrong somewhere, possibly
>> a buffer overrun or yet some memory not properly set.
> 
> How do we reproduce?

make fate-lavfi -j is enough

> What kind of msgs do you get in valgrind?

you will get a warning about uninitialized data if you are lucky
(apparently valgrind and asan have hard time spotting it)

> Do 32/64bit outputs match?

Multiple run end up with different outputs. I don't have a 32bit system.
So far I tested on Linux and I can test on OSX later.

lu
___
libav-devel mailing list
libav-devel@libav.org
https://lists.libav.org/mailman/listinfo/libav-devel

Re: [libav-devel] [PATCH 2/2] swscale: support gray to 9bit and 10bit formats

[Ronald S. Bultje]

Hi,

On Fri, Oct 26, 2012 at 8:35 AM, Luca Barbato  wrote:
> Signed-off-by: Luca Barbato 
> ---
>
> Partially fixes the issue, there is still something wrong somewhere, possibly
> a buffer overrun or yet some memory not properly set.

How do we reproduce? What kind of msgs do you get in valgrind? Do
32/64bit outputs match?

Ronald
___
libav-devel mailing list
libav-devel@libav.org
https://lists.libav.org/mailman/listinfo/libav-devel

[libav-devel] [PATCH 2/2] swscale: support gray to 9bit and 10bit formats

[Luca Barbato]

Signed-off-by: Luca Barbato 
---

Partially fixes the issue, there is still something wrong somewhere, possibly
a buffer overrun or yet some memory not properly set.

 libswscale/swscale.c  | 38 --
 libswscale/swscale_unscaled.c | 32 ++--
 tests/ref/lavfi/pixdesc   | 18 ++
 tests/ref/lavfi/pixfmts_copy  | 18 ++
 tests/ref/lavfi/pixfmts_null  | 18 ++
 tests/ref/lavfi/pixfmts_scale | 18 ++
 tests/ref/lavfi/pixfmts_vflip | 18 ++
 7 files changed, 156 insertions(+), 4 deletions(-)

diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 3f54e4d..07dd582 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -61,6 +61,28 @@ static av_always_inline void fillPlane(uint8_t *plane, int 
stride, int width,
 }
 }
 
+static void fill_plane9or10(uint8_t *plane, int stride, int width,
+int height, int y, uint8_t val,
+const int dst_depth, const int big_endian)
+{
+int i, j;
+uint16_t *dst = (uint16_t *) plane;
+#define FILL8TO9_OR_10(wfunc) \
+for (i = 0; i < height; i++) { \
+for (j = 0; j < width; j++) { \
+wfunc(&dst[j], (val << (dst_depth - 8)) |  \
+   (val >> (16 - dst_depth))); \
+} \
+dst += stride / 2; \
+}
+if (big_endian) {
+FILL8TO9_OR_10(AV_WB16);
+} else {
+FILL8TO9_OR_10(AV_WL16);
+}
+}
+
+
 static void hScale16To19_c(SwsContext *c, int16_t *_dst, int dstW,
const uint8_t *_src, const int16_t *filter,
const int32_t *filterPos, int filterSize)
@@ -658,8 +680,20 @@ static int swScale(SwsContext *c, const uint8_t *src[],
 }
 }
 
-if (isPlanar(dstFormat) && isALPHA(dstFormat) && !alpPixBuf)
-fillPlane(dst[3], dstStride[3], dstW, dstY - lastDstY, lastDstY, 255);
+if (isPlanar(dstFormat) && isALPHA(dstFormat) && !alpPixBuf) {
+int length = dstW;
+int height = dstY - lastDstY;
+if (is16BPS(c->dstFormat))
+length *= 2;
+
+if (is9_OR_10BPS(dstFormat)) {
+const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(dstFormat);
+fill_plane9or10(dst[3], dstStride[3], length, height, lastDstY,
+255, desc->comp[3].depth_minus1 + 1,
+isBE(dstFormat));
+} else
+fillPlane(dst[3], dstStride[3], length, height, lastDstY, 255);
+}
 
 #if HAVE_MMXEXT_INLINE
 if (av_get_cpu_flags() & AV_CPU_FLAG_MMXEXT)
diff --git a/libswscale/swscale_unscaled.c b/libswscale/swscale_unscaled.c
index 5efc647..ade0b07 100644
--- a/libswscale/swscale_unscaled.c
+++ b/libswscale/swscale_unscaled.c
@@ -98,6 +98,27 @@ static void fillPlane(uint8_t *plane, int stride, int width, 
int height, int y,
 }
 }
 
+static void fill_plane9or10(uint8_t *plane, int stride, int width,
+int height, int y, uint8_t val,
+const int dst_depth, const int big_endian)
+{
+int i, j;
+uint16_t *dst = (uint16_t *) plane;
+#define FILL8TO9_OR_10(wfunc) \
+for (i = 0; i < height; i++) { \
+for (j = 0; j < width; j++) { \
+wfunc(&dst[j], (val << (dst_depth - 8)) |  \
+   (val >> (16 - dst_depth))); \
+} \
+dst += stride / 2; \
+}
+if (big_endian) {
+FILL8TO9_OR_10(AV_WB16);
+} else {
+FILL8TO9_OR_10(AV_WL16);
+}
+}
+
 static void copyPlane(const uint8_t *src, int srcStride,
   int srcSliceY, int srcSliceH, int width,
   uint8_t *dst, int dstStride)
@@ -677,10 +698,17 @@ static int planarCopyWrapper(SwsContext *c, const uint8_t 
*src[],
 // ignore palette for GRAY8
 if (plane == 1 && !dst[2]) continue;
 if (!src[plane] || (plane == 1 && !src[2])) {
+int val = (plane == 3) ? 255 : 128;
 if (is16BPS(c->dstFormat))
 length *= 2;
-fillPlane(dst[plane], dstStride[plane], length, height, y,
-  (plane == 3) ? 255 : 128);
+if (is9_OR_10BPS(c->dstFormat)) {
+fill_plane9or10(dst[plane], dstStride[plane],
+length, height, y, val,
+desc_dst->comp[plane].depth_minus1 + 1,
+isBE(c->dstFormat));
+} else
+fillPlane(dst[plane], dstStride[plane], length, height, y,
+  val);
 } else {
 if (is9_OR_10BPS(c->srcFormat)) {
 const int src_depth = desc_src->comp[plane].depth_minus1 + 1;
diff --git a/tests/ref/lavfi/pixdesc b/tests/ref/lavfi/pixdesc
index dc1b857..6669efb 100644
--- a/tests/ref/lavfi/pixdesc
+++ b/tests/ref/lavfi/pixdesc
@@