PR #25891 opened by Rainbaby
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25891
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25891.patch

The CAVS inverse transform skipped the 16 bit clip that GB/T 20090.2
and GY/T 257.1 specify, and the SSE2 version wrapped in 16 bit lanes,
so C and SSE2 could disagree even on conformant input.

I found it on AVS+ broadcasts with weighted quantization (support
follows in a separate PR): SSE2 differed from the reference decoder in
46 of 3734 frames. With these patches C and SSE2 match it on all of
them.

x86_32 now uses the C version, about 8% slower decoding. On x86_64 the
SSE2 transform goes from 35 to 56 cycles a call (C is about 240).


>From 3dc9865c613081a6c55641b43e2bf77ce93d4460 Mon Sep 17 00:00:00 2001
From: Rainbaby <[email protected]>
Date: Tue, 3 Nov 2026 19:20:48 +0900
Subject: [PATCH 1/3] avcodec/cavsdsp: clip the inverse transform as specified

GB/T 20090.2 9.2 and GY/T 257.1 9.7 clip both passes to 16 bits; the C
code clipped neither, so some conformant blocks came out wrong. The
second clip is already done by av_clip_uint8(). The rounding bias moves
into the second pass so the first pass clips what the standard clips.

Signed-off-by: Rainbaby <[email protected]>
---
 libavcodec/cavsdsp.c | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/libavcodec/cavsdsp.c b/libavcodec/cavsdsp.c
index 8b2fa2dde90..619dba0f3b6 100644
--- a/libavcodec/cavsdsp.c
+++ b/libavcodec/cavsdsp.c
@@ -196,8 +196,6 @@ static void cavs_idct8_add_c(uint8_t *dst, int16_t *block, 
ptrdiff_t stride)
     int i;
     int16_t (*src)[8] = (int16_t(*)[8])block;
 
-    src[0][0] += 8;
-
     for( i = 0; i < 8; i++ ) {
         const int a0 = 3 * src[i][1] - 2 * src[i][7];
         const int a1 = 3 * src[i][3] + 2 * src[i][5];
@@ -219,14 +217,14 @@ static void cavs_idct8_add_c(uint8_t *dst, int16_t 
*block, ptrdiff_t stride)
         const int b2 = a5 - a7;
         const int b3 = a4 - a6;
 
-        src[i][0] = (b0 + b4) >> 3;
-        src[i][1] = (b1 + b5) >> 3;
-        src[i][2] = (b2 + b6) >> 3;
-        src[i][3] = (b3 + b7) >> 3;
-        src[i][4] = (b3 - b7) >> 3;
-        src[i][5] = (b2 - b6) >> 3;
-        src[i][6] = (b1 - b5) >> 3;
-        src[i][7] = (b0 - b4) >> 3;
+        src[i][0] = av_clip_int16(b0 + b4) >> 3;
+        src[i][1] = av_clip_int16(b1 + b5) >> 3;
+        src[i][2] = av_clip_int16(b2 + b6) >> 3;
+        src[i][3] = av_clip_int16(b3 + b7) >> 3;
+        src[i][4] = av_clip_int16(b3 - b7) >> 3;
+        src[i][5] = av_clip_int16(b2 - b6) >> 3;
+        src[i][6] = av_clip_int16(b1 - b5) >> 3;
+        src[i][7] = av_clip_int16(b0 - b4) >> 3;
     }
     for( i = 0; i < 8; i++ ) {
         const int a0 = 3 * src[1][i] - 2 * src[7][i];
@@ -241,8 +239,8 @@ static void cavs_idct8_add_c(uint8_t *dst, int16_t *block, 
ptrdiff_t stride)
 
         const int a7 = 4 * src[2][i] - 10 * src[6][i];
         const int a6 = 4 * src[6][i] + 10 * src[2][i];
-        const int a5 = 8 * (src[0][i] - src[4][i]);
-        const int a4 = 8 * (src[0][i] + src[4][i]);
+        const int a5 = 8 * (src[0][i] - src[4][i]) + 64;
+        const int a4 = 8 * (src[0][i] + src[4][i]) + 64;
 
         const int b0 = a4 + a6;
         const int b1 = a5 + a7;
-- 
2.54.0


>From 2a4cc6b131c69870b9f99ff9e6d1791930ce047f Mon Sep 17 00:00:00 2001
From: Rainbaby <[email protected]>
Date: Tue, 3 Nov 2026 19:20:48 +0900
Subject: [PATCH 2/3] avcodec/x86/cavsidct: fix overflow on large coefficients

The SSE2 code worked in 16 bit lanes, so sums past int16_t wrapped. Use
32 bit lanes and clip the first pass like the C code. This needs 16 xmm
registers, so x86_32 now uses the C version.

Signed-off-by: Rainbaby <[email protected]>
---
 libavcodec/x86/cavsdsp.c    |   4 +
 libavcodec/x86/cavsidct.asm | 271 +++++++++++++++++++-----------------
 2 files changed, 151 insertions(+), 124 deletions(-)

diff --git a/libavcodec/x86/cavsdsp.c b/libavcodec/x86/cavsdsp.c
index a26fc9661c6..fa24f416e82 100644
--- a/libavcodec/x86/cavsdsp.c
+++ b/libavcodec/x86/cavsdsp.c
@@ -35,6 +35,7 @@
 
 #if HAVE_SSE2_EXTERNAL
 
+#if ARCH_X86_64
 void ff_cavs_idct8_sse2(int16_t *out, const int16_t *in);
 
 static void cavs_idct8_add_sse2(uint8_t *dst, int16_t *block, ptrdiff_t stride)
@@ -43,6 +44,7 @@ static void cavs_idct8_add_sse2(uint8_t *dst, int16_t *block, 
ptrdiff_t stride)
     ff_cavs_idct8_sse2(b2, block);
     ff_add_pixels_clamped_sse2(b2, dst, stride);
 }
+#endif
 
 #define DEF_QPEL(OPNAME) \
     void ff_ ## OPNAME ## _cavs_qpel8_mc20_sse2(uint8_t *dst, const uint8_t 
*src, ptrdiff_t stride);     \
@@ -112,8 +114,10 @@ av_cold void ff_cavsdsp_init_x86(CAVSDSPContext *c)
         c->avg_cavs_qpel_pixels_tab[1][ 8] = ff_avg_cavs_qpel8_mc02_sse2;
         c->avg_cavs_qpel_pixels_tab[1][12] = ff_avg_cavs_qpel8_mc03_sse2;
 
+#if ARCH_X86_64
         c->cavs_idct8_add = cavs_idct8_add_sse2;
         c->idct_perm      = FF_IDCT_PERM_TRANSPOSE;
+#endif
     }
 #endif
 }
diff --git a/libavcodec/x86/cavsidct.asm b/libavcodec/x86/cavsidct.asm
index a0af002a787..2a659e2038d 100644
--- a/libavcodec/x86/cavsidct.asm
+++ b/libavcodec/x86/cavsidct.asm
@@ -24,141 +24,164 @@
 
 %include "libavutil/x86/x86util.asm"
 
-cextern pw_4
-cextern pw_64
+%if ARCH_X86_64
+SECTION_RODATA
+
+; coefficient pairs for pmaddwd on interleaved (src1, src7), (src3, src5),
+; (src0, src4) and (src2, src6)
+pw_10_2:    times 4 dw  10,   2
+pw_9_6:     times 4 dw   9,   6
+pw_9_m6:    times 4 dw   9,  -6
+pw_m2_m10:  times 4 dw  -2, -10
+pw_6_9:     times 4 dw   6,   9
+pw_m10_2:   times 4 dw -10,   2
+pw_2_m10:   times 4 dw   2, -10
+pw_m6_9:    times 4 dw  -6,   9
+pw_8_8:     times 4 dw   8,   8
+pw_8_m8:    times 4 dw   8,  -8
+pw_10_4:    times 4 dw  10,   4
+pw_4_m10:   times 4 dw   4, -10
+pd_4:       times 4 dd 4
+
+cextern pd_64
 
 SECTION .text
 
-%macro CAVS_IDCT8_1D 2-3 1 ; source, round, init_load
-%if %3 == 1
-    mova            m4, [%1+7*16]       ; m4 = src7
-    mova            m5, [%1+1*16]       ; m5 = src1
-    mova            m2, [%1+5*16]       ; m2 = src5
-    mova            m7, [%1+3*16]       ; m7 = src3
-%else
-    SWAP             1, 7
-    SWAP             4, 6
-%endif
-    mova            m0, m4
-    mova            m3, m5
-    mova            m6, m2
-    mova            m1, m7
+; One 1D transform of four columns in 32 bit, so no intermediate wraps.
+; %1 = l or h (columns 0-3 or 4-7), %2 = rows of 8 coefficients, %3 = bias.
+; Out: m4, m5, m6, m0, m2, m1, m8, m3 = dst0 to dst7 as dwords.
+%macro CAVS_IDCT8_1D_HALF 3
+    mova            m0, [%2+1*16]
+    mova            m1, [%2+3*16]
+    mova            m2, [%2+0*16]
+    mova            m3, [%2+2*16]
+    punpck%1wd      m0, [%2+7*16]       ; m0 = src1, src7
+    punpck%1wd      m1, [%2+5*16]       ; m1 = src3, src5
+    punpck%1wd      m2, [%2+4*16]       ; m2 = src0, src4
+    punpck%1wd      m3, [%2+6*16]       ; m3 = src2, src6
 
-    paddw           m4, m4              ; m4 = 2*src7
-    paddw           m3, m3              ; m3 = 2*src1
-    paddw           m6, m6              ; m6 = 2*src5
-    paddw           m1, m1              ; m1 = 2*src3
-    paddw           m0, m4              ; m0 = 3*src7
-    paddw           m5, m3              ; m5 = 3*src1
-    paddw           m2, m6              ; m2 = 3*src5
-    paddw           m7, m1              ; m7 = 3*src3
-    psubw           m5, m4              ; m5 = 3*src1 - 2*src7 = a0
-    paddw           m7, m6              ; m7 = 3*src3 - 2*src5 = a1
-    psubw           m1, m2              ; m1 = 2*src3 - 3*src5 = a2
-    paddw           m3, m0              ; m3 = 2*src1 - 3*src7 = a3
+    mova            m4, m0
+    mova            m9, m1
+    pmaddwd         m4, [pw_10_2]
+    pmaddwd         m9, [pw_9_6]
+    paddd           m4, m9              ; m4 = b4
+    mova            m5, m0
+    mova            m9, m1
+    pmaddwd         m5, [pw_9_m6]
+    pmaddwd         m9, [pw_m2_m10]
+    paddd           m5, m9              ; m5 = b5
+    mova            m6, m0
+    mova            m9, m1
+    pmaddwd         m6, [pw_6_9]
+    pmaddwd         m9, [pw_m10_2]
+    paddd           m6, m9              ; m6 = b6
+    pmaddwd         m0, [pw_2_m10]
+    pmaddwd         m1, [pw_m6_9]
+    paddd           m0, m1              ; m0 = b7
 
-    mova            m4, m5
-    mova            m6, m7
-    mova            m0, m3
-    mova            m2, m1
-    paddw           m7, m5              ; m7 = a0 + a1
-    psubw           m5, m6              ; m5 = a0 - a1
-    paddw           m7, m3              ; m7 = a0 + a1 + a3
-    paddw           m5, m1              ; m5 = a0 - a1 + a2
-    paddw           m7, m7
-    paddw           m5, m5
-    paddw           m7, m6              ; m7 = b4
-    paddw           m5, m4              ; m5 = b5
+    mova            m1, m2
+    pmaddwd         m2, [pw_8_8]
+    pmaddwd         m1, [pw_8_m8]
+    paddd           m2, %3              ; m2 = a4
+    paddd           m1, %3              ; m1 = a5
+    mova            m8, m3
+    pmaddwd         m3, [pw_10_4]       ; m3 = a6
+    pmaddwd         m8, [pw_4_m10]      ; m8 = a7
 
-    paddw           m1, m3              ; m1 = a3 + a2
-    psubw           m3, m2              ; m3 = a3 - a2
-    psubw           m4, m1              ; m4 = a0 - a2 - a3
-    mova            m1, m4              ; m1 = a0 - a2 - a3
-    psubw           m3, m6              ; m3 = a3 - a2 - a1
-    paddw           m1, m1
-    paddw           m3, m3
-    psubw           m1, m2              ; m1 = b7
-    paddw           m3, m0              ; m3 = b6
+    SUMSUB_BA    d, 3, 2, 9             ; m3 = b0, m2 = b3
+    SUMSUB_BA    d, 8, 1, 9             ; m8 = b1, m1 = b2
+    SUMSUB_BA    d, 4, 3, 9             ; m4 = dst0, m3 = dst7
+    SUMSUB_BA    d, 5, 8, 9             ; m5 = dst1, m8 = dst6
+    SUMSUB_BA    d, 6, 1, 9             ; m6 = dst2, m1 = dst5
+    SUMSUB_BA    d, 0, 2, 9             ; m0 = dst3, m2 = dst4
+%endmacro
 
-    mova            m2, [%1+2*16]       ; m2 = src2
-    mova            m6, [%1+6*16]       ; m6 = src6
-    mova            m4, m2
-    mova            m0, m6
-    psllw           m4, 2               ; m4 = 4*src2
-    psllw           m6, 2               ; m6 = 4*src6
-    paddw           m2, m4              ; m2 = 5*src2
-    paddw           m0, m6              ; m0 = 5*src6
-    paddw           m2, m2
-    paddw           m0, m0
-    psubw           m4, m0              ; m4 = 4*src2 - 10*src6 = a7
-    paddw           m6, m2              ; m6 = 4*src6 + 10*src2 = a6
+; store the eight dword outputs of a left half
+%macro CAVS_STORE_HALF 1
+    mova    [%1+0*16], m4
+    mova    [%1+1*16], m5
+    mova    [%1+2*16], m6
+    mova    [%1+3*16], m0
+    mova    [%1+4*16], m2
+    mova    [%1+5*16], m1
+    mova    [%1+6*16], m8
+    mova    [%1+7*16], m3
+%endmacro
 
-    mova            m2, [%1+0*16]       ; m2 = src0
-    mova            m0, [%1+4*16]       ; m0 = src4
-%if ARCH_X86_64
-    SUMSUB_BA     w, 0, 2, 8            ; m0 = src0 + src4, m2 = src0 - src4
-%else
-    SUMSUB_BA     w, 0, 2               ; m0 = src0 + src4, m2 = src0 - src4
-%endif
-    psllw           m0, 3
-    psllw           m2, 3
-    paddw           m0, %2              ; add rounding bias
-    paddw           m2, %2              ; add rounding bias
-
-%if ARCH_X86_64
-    SUMSUB_BA     w, 6, 0, 8            ; m6 = a4 + a6, m0 = a4 - a6
-    SUMSUB_BA     w, 4, 2, 8            ; m4 = a5 + a7, m2 = a5 - a7
-    SUMSUB_BA     w, 7, 6, 8            ; m7 = dst0, m6 = dst7
-    SUMSUB_BA     w, 5, 4, 8            ; m5 = dst1, m4 = dst6
-    SUMSUB_BA     w, 3, 2, 8            ; m3 = dst2, m2 = dst5
-    SUMSUB_BA     w, 1, 0, 8            ; m1 = dst3, m0 = dst4
-%else
-    SUMSUB_BA     w, 6, 0               ; m6 = a4 + a6, m0 = a4 - a6
-    SUMSUB_BA     w, 4, 2               ; m4 = a5 + a7, m2 = a5 - a7
-    SUMSUB_BA     w, 7, 6               ; m7 = dst0, m6 = dst7
-    SUMSUB_BA     w, 5, 4               ; m5 = dst1, m4 = dst6
-    SUMSUB_BA     w, 3, 2               ; m3 = dst2, m2 = dst5
-    SUMSUB_BA     w, 1, 0               ; m1 = dst3, m0 = dst4
-%endif
+; join a stored left half with the right half in registers, saturating
+; to int16; out: m10 to m15, m7, m9 = dst0 to dst7
+%macro CAVS_PACK_HALVES 1
+    mova           m10, [%1+0*16]
+    mova           m11, [%1+1*16]
+    mova           m12, [%1+2*16]
+    mova           m13, [%1+3*16]
+    mova           m14, [%1+4*16]
+    mova           m15, [%1+5*16]
+    mova            m7, [%1+6*16]
+    mova            m9, [%1+7*16]
+    packssdw       m10, m4
+    packssdw       m11, m5
+    packssdw       m12, m6
+    packssdw       m13, m0
+    packssdw       m14, m2
+    packssdw       m15, m1
+    packssdw        m7, m8
+    packssdw        m9, m3
 %endmacro
 
 INIT_XMM sse2
-cglobal cavs_idct8, 2, 2, 8 + ARCH_X86_64, 0 - 8 * 16, out, in
-    CAVS_IDCT8_1D  inq, [pw_4]
+cglobal cavs_idct8, 2, 2, 16, 0 - 16 * 16, out, in
+    ; first pass: Clip3(-2^15, 2^15 - 1, x + 4) >> 3, the clip by packssdw
+    CAVS_IDCT8_1D_HALF l, inq, [pd_4]
+    CAVS_STORE_HALF rsp
+    CAVS_IDCT8_1D_HALF h, inq, [pd_4]
+    CAVS_PACK_HALVES rsp
+    psraw          m10, 3
+    psraw          m11, 3
+    psraw          m12, 3
+    psraw          m13, 3
+    psraw          m14, 3
+    psraw          m15, 3
     psraw           m7, 3
-    psraw           m6, 3
-    psraw           m5, 3
-    psraw           m4, 3
-    psraw           m3, 3
-    psraw           m2, 3
-    psraw           m1, 3
-    psraw           m0, 3
-%if ARCH_X86_64
-    TRANSPOSE8x8W    7, 5, 3, 1, 0, 2, 4, 6, 8
-    mova    [rsp+4*16], m0
-%else
-    mova    [rsp+0*16], m4
-    TRANSPOSE8x8W    7, 5, 3, 1, 0, 2, 4, 6, [rsp+0*16], [rsp+4*16], 1
-%endif
-    mova    [rsp+0*16], m7
-    mova    [rsp+2*16], m3
-    mova    [rsp+6*16], m4
-    CAVS_IDCT8_1D  rsp, [pw_64], 0
-    psraw           m7, 7
-    psraw           m6, 7
-    psraw           m5, 7
-    psraw           m4, 7
-    psraw           m3, 7
-    psraw           m2, 7
-    psraw           m1, 7
-    psraw           m0, 7
+    psraw           m9, 3
+    TRANSPOSE8x8W   10, 11, 12, 13, 14, 15, 7, 9, 0
+    mova    [rsp+ 8*16], m10
+    mova    [rsp+ 9*16], m11
+    mova    [rsp+10*16], m12
+    mova    [rsp+11*16], m13
+    mova    [rsp+12*16], m14
+    mova    [rsp+13*16], m15
+    mova    [rsp+14*16], m7
+    mova    [rsp+15*16], m9
 
-    mova   [outq+0*16], m7
-    mova   [outq+1*16], m5
-    mova   [outq+2*16], m3
-    mova   [outq+3*16], m1
-    mova   [outq+4*16], m0
-    mova   [outq+5*16], m2
-    mova   [outq+6*16], m4
-    mova   [outq+7*16], m6
+    ; second pass: (x + 64) >> 7, which fits int16 for any int16 input
+    CAVS_IDCT8_1D_HALF l, rsp+8*16, [pd_64]
+    psrad           m4, 7
+    psrad           m5, 7
+    psrad           m6, 7
+    psrad           m0, 7
+    psrad           m2, 7
+    psrad           m1, 7
+    psrad           m8, 7
+    psrad           m3, 7
+    CAVS_STORE_HALF rsp
+    CAVS_IDCT8_1D_HALF h, rsp+8*16, [pd_64]
+    psrad           m4, 7
+    psrad           m5, 7
+    psrad           m6, 7
+    psrad           m0, 7
+    psrad           m2, 7
+    psrad           m1, 7
+    psrad           m8, 7
+    psrad           m3, 7
+    CAVS_PACK_HALVES rsp
+    mova   [outq+0*16], m10
+    mova   [outq+1*16], m11
+    mova   [outq+2*16], m12
+    mova   [outq+3*16], m13
+    mova   [outq+4*16], m14
+    mova   [outq+5*16], m15
+    mova   [outq+6*16], m7
+    mova   [outq+7*16], m9
     RET
+%endif ; ARCH_X86_64
-- 
2.54.0


>From 4d7fe4db74cc9ea73bdd6fbb9d5b9e2859e4b662 Mon Sep 17 00:00:00 2001
From: Rainbaby <[email protected]>
Date: Tue, 3 Nov 2026 19:20:48 +0900
Subject: [PATCH 3/3] checkasm/cavsdsp: test cavs_idct8_add

Coefficients span the whole int16_t range, so both the first pass clip
and second pass sums past int16_t are reached. The SSE2 version fails
this without the previous commit.

Signed-off-by: Rainbaby <[email protected]>
---
 tests/checkasm/cavsdsp.c | 51 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/tests/checkasm/cavsdsp.c b/tests/checkasm/cavsdsp.c
index 47c2510687f..cb40b7e0453 100644
--- a/tests/checkasm/cavsdsp.c
+++ b/tests/checkasm/cavsdsp.c
@@ -21,10 +21,12 @@
 #include <string.h>
 
 #include "checkasm.h"
+#include "libavutil/common.h"
 #include "libavutil/intreadwrite.h"
 #include "libavutil/macros.h"
 #include "libavutil/mem_internal.h"
 #include "libavcodec/cavsdsp.h"
+#include "libavcodec/idctdsp.h"
 
 
 enum {
@@ -114,8 +116,57 @@ static void check_cavs_qpeldsp(void)
     }
 }
 
+static void check_cavs_idct8_add(void)
+{
+    DECLARE_ALIGNED(16, int16_t, block0)[64];
+    DECLARE_ALIGNED(16, int16_t, block1)[64];
+    DECLARE_ALIGNED(16, uint8_t, dstbuf0)[8 * 16];
+    DECLARE_ALIGNED(16, uint8_t, dstbuf1)[8 * 16];
+    CAVSDSPContext cavsdsp;
+    declare_func(void, uint8_t *dst, int16_t *block, ptrdiff_t stride);
+
+    ff_cavsdsp_init(&cavsdsp);
+
+    if (check_func(cavsdsp.cavs_idct8_add, "cavs_idct8_add")) {
+        for (int iter = 0; iter < 64; ++iter) {
+            int16_t coef[64] = { 0 };
+            /*
+             * The C code clips the first pass as clause 9.7 does and is
+             * defined for any int16_t input, so the SIMD must match it
+             * everywhere. The random shift spreads magnitudes from the
+             * full int16_t range down to single digits, reaching both the
+             * first pass clip and second pass sums past int16_t.
+             */
+            int n = 1 + rnd() % 64;
+
+            for (int k = 0; k < n; ++k)
+                coef[rnd() % 64] = (int16_t)rnd() >> (rnd() % 16);
+
+            /* each implementation reads the block in its scan permutation */
+            for (int i = 0; i < 8; ++i)
+                for (int j = 0; j < 8; ++j) {
+                    block0[8 * i + j] = coef[8 * i + j];
+                    if (cavsdsp.idct_perm == FF_IDCT_PERM_TRANSPOSE)
+                        block1[8 * j + i] = coef[8 * i + j];
+                    else
+                        block1[8 * i + j] = coef[8 * i + j];
+                }
+
+            randomize_buffers(dstbuf0, dstbuf1);
+            call_ref(dstbuf0, block0, 16);
+            call_new(dstbuf1, block1, 16);
+            /* the block is caller cleared scratch, only the dst counts */
+            if (memcmp(dstbuf0, dstbuf1, sizeof(dstbuf0)))
+                fail();
+        }
+        bench_new(dstbuf1, block1, 16);
+    }
+}
+
 void checkasm_check_cavsdsp(void)
 {
     check_cavs_qpeldsp();
     report("qpeldsp");
+    check_cavs_idct8_add();
+    report("idct8");
 }
-- 
2.54.0

_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]

Reply via email to