PR #24587 opened by Marcos Ashton (MarcosAsh) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24587 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24587.patch
AVX2 planar intra prediction for VVC, 8 bit. VVC intra prediction had no asm on any arch, so this also brings along the checkasm test from #22808. The test is Hankang Li's, squashed into one commit with his authorship kept. I asked on #22808 whether it was still being worked on and got no answer, so I am carrying it. @authentisity, if you would rather land it yourself just say so and I will rebase onto yours. I fixed the buffer overrun the Fairy found there plus a couple of small things, all listed in the commit message. Only planar has asm so far, so the other checks do not run yet. I did run them once through C wrappers under ASan to make sure they work. The asm stays in 16 bit words. Pulling 2^min(logw,logh) out of the numerator means it never goes above 511*64 = 32704. The part that is linear in y is a running accumulator, so a row is one pmullw and three shifts per group of columns. Widths 32 and 64 do two groups at a time and write the row with one 32 byte store. The height is a run time argument like in the C code, which covers the 1 and 2 high blocks that ISP and chroma produce. This touches the same wiring lines as #24522. I will rebase whichever goes in second. checkasm passes on all 35 shapes from 4x1 to 64x64, 200 seeds. Core Ultra 7 155H: ``` vvc_pred_planar_4x4_8_c: 63.0 vvc_pred_planar_4x4_8_avx2: 20.3 ( 3.10x) vvc_pred_planar_8x8_8_c: 216.7 vvc_pred_planar_8x8_8_avx2: 35.6 ( 6.09x) vvc_pred_planar_16x16_8_c: 966.4 vvc_pred_planar_16x16_8_avx2: 80.3 (12.03x) vvc_pred_planar_32x32_8_c: 3032.7 vvc_pred_planar_32x32_8_avx2: 251.5 (12.06x) vvc_pred_planar_64x64_8_c: 10389.9 vvc_pred_planar_64x64_8_avx2: 967.1 (10.74x) ``` >From b264286cfa1028a9113b63c021cba3cebae67b8a Mon Sep 17 00:00:00 2001 From: Hankang Li <[email protected]> Date: Mon, 14 Sep 2026 11:06:24 +0100 Subject: [PATCH 1/2] tests/checkasm: Add checkasm test for vvc intra prediction Covers pred_dc, pred_planar, pred_v, pred_h, pred_mip and both pred_angular functions for every block shape from 4x4 to 64x64 at 8, 10 and 12 bit. Only pred_planar has asm so far, added in the next commit; the other checks are skipped until there is asm for them. This is the test from #22808, squashed. [Marcos Ashton Iglesias: size top_buf and left_buf for 16 bit samples, 778 bytes rather than 405, which randomize_buffers() overran. Declare bit_depth in its loop, always set mode_category, and test pred_planar down to a height of 1, which ISP splits and chroma produce.] --- tests/checkasm/Makefile | 2 +- tests/checkasm/checkasm.c | 1 + tests/checkasm/checkasm.h | 1 + tests/checkasm/vvc_pred.c | 295 ++++++++++++++++++++++++++++++++++++++ tests/fate/checkasm.mak | 1 + 5 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 tests/checkasm/vvc_pred.c diff --git a/tests/checkasm/Makefile b/tests/checkasm/Makefile index 6b78f0d35a..f0383f4c04 100644 --- a/tests/checkasm/Makefile +++ b/tests/checkasm/Makefile @@ -62,7 +62,7 @@ AVCODECOBJS-$(CONFIG_V210_ENCODER) += v210enc.o AVCODECOBJS-$(CONFIG_VORBIS_DECODER) += vorbisdsp.o AVCODECOBJS-$(CONFIG_VP6_DECODER) += vp6dsp.o AVCODECOBJS-$(CONFIG_VP9_DECODER) += vp9dsp.o -AVCODECOBJS-$(CONFIG_VVC_DECODER) += vvc_alf.o vvc_mc.o vvc_sao.o +AVCODECOBJS-$(CONFIG_VVC_DECODER) += vvc_alf.o vvc_mc.o vvc_pred.o vvc_sao.o CHECKASMOBJS-$(CONFIG_AVCODEC) += $(AVCODECOBJS-yes) diff --git a/tests/checkasm/checkasm.c b/tests/checkasm/checkasm.c index 5127b35729..5e7c11787f 100644 --- a/tests/checkasm/checkasm.c +++ b/tests/checkasm/checkasm.c @@ -238,6 +238,7 @@ static const CheckasmTest tests[] = { #if CONFIG_VVC_DECODER { "vvc_alf", checkasm_check_vvc_alf }, { "vvc_mc", checkasm_check_vvc_mc }, + { "vvc_pred", checkasm_check_vvc_pred }, { "vvc_sao", checkasm_check_vvc_sao }, #endif #endif diff --git a/tests/checkasm/checkasm.h b/tests/checkasm/checkasm.h index 3e975e54a7..783ad6d982 100644 --- a/tests/checkasm/checkasm.h +++ b/tests/checkasm/checkasm.h @@ -128,6 +128,7 @@ void checkasm_check_videodsp(void); void checkasm_check_vorbisdsp(void); void checkasm_check_vvc_alf(void); void checkasm_check_vvc_mc(void); +void checkasm_check_vvc_pred(void); void checkasm_check_vvc_sao(void); void checkasm_uninit_crc(void); diff --git a/tests/checkasm/vvc_pred.c b/tests/checkasm/vvc_pred.c new file mode 100644 index 0000000000..3229bba858 --- /dev/null +++ b/tests/checkasm/vvc_pred.c @@ -0,0 +1,295 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with FFmpeg; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <string.h> +#include "checkasm.h" +#include "libavcodec/vvc/dsp.h" +#include "libavcodec/vvc/intra.h" +#include "libavutil/intreadwrite.h" +#include "libavutil/mem_internal.h" + +static const uint32_t pixel_mask[3] = { 0xffffffff, 0x03ff03ff, 0x0fff0fff }; +static const int mip_modes[] = {16, 8, 6}; + +#define SIZEOF_PIXEL ((bit_depth + 7) / 8) +#define PIXEL_STRIDE (MAX_TB_SIZE * 2) // 2 * max transform size +#define BUF_SIZE (2 * PIXEL_STRIDE * 64) // Enough for 64x64 with stride=128 +// IntraEdgeParams arrays are uint16_t, rounded up as they are filled 4 bytes at a time +#define REF_BYTES FFALIGN(2 * (6 * MAX_TB_SIZE + 5), 4) +#define REF_OFFSET (MAX_TB_SIZE + 3) // Same as offset from prepare_intra_edge_params + +#define randomize_buffers() \ + do { \ + uint32_t mask = pixel_mask[(bit_depth - 8) >> 1]; \ + for (int i = 0; i < BUF_SIZE; i += 4) { \ + uint32_t r = rnd() & mask; \ + AV_WN32A(buf0 + i, r); \ + AV_WN32A(buf1 + i, r); \ + } \ + /* Make sure reference buffer writes don't go out of bounds */ \ + for (int i = -(REF_OFFSET * SIZEOF_PIXEL); \ + i < REF_BYTES - REF_OFFSET * SIZEOF_PIXEL; i += 4) { \ + uint32_t r = rnd() & mask; \ + AV_WN32A(top + i, r); \ + AV_WN32A(left + i, r); \ + } \ + } while (0) + +// From prepare_intra_edge_params in avcodec/vvc/intra_template.c +static int derive_filter_flag(const int mode, const int ref_idx, const int w, const int h) +{ + if (ff_vvc_ref_filter_flag_derive(mode) || ref_idx) + return 0; + const int min_dist_ver_hor = FFMIN(abs(mode - 50), abs(mode - 18)); + const int intra_hor_ver_dist_thres[] = {24, 14, 2, 0, 0}; + const int ntbs = (av_log2(w) + av_log2(h)) >> 1; + return min_dist_ver_hor > intra_hor_ver_dist_thres[ntbs - 2]; +} + +static void check_pred_dc(VVCDSPContext *c, + uint8_t *buf0, uint8_t *buf1, + uint8_t *top, uint8_t *left, int bit_depth) +{ + declare_func(void, uint8_t *src, const uint8_t *top, + const uint8_t *left, const int w, const int h, const ptrdiff_t stride); + + /* Test all sizes 4x4 up to 64x64, including non-square */ + for (int log2_w = 2; log2_w <= 6; log2_w++) { + for (int log2_h = 2; log2_h <= 6; log2_h++) { + int w = 1 << log2_w; + int h = 1 << log2_h; + ptrdiff_t stride = FFALIGN((w + rnd() % 64), 32); + + if (check_func(c->intra.pred_dc, "vvc_pred_dc_%dx%d_%d", + w, h, bit_depth)) { + randomize_buffers(); + call_ref(buf0, top, left, w, h, stride); + call_new(buf1, top, left, w, h, stride); + if (memcmp(buf0, buf1, BUF_SIZE)) + fail(); + if (w == h) + bench_new(buf1, top, left, w, h, stride); + } + } + } + + report("pred_dc"); +} + +static void check_pred_planar(VVCDSPContext *c, + uint8_t *buf0, uint8_t *buf1, + uint8_t *top, uint8_t *left, int bit_depth) +{ + declare_func(void, uint8_t *src, const uint8_t *top, + const uint8_t *left, const int w, const int h, const ptrdiff_t stride); + + /* Test all sizes 4x1 up to 64x64, including non-square. Heights of 1 and 2 + * come from ISP splits and from chroma. */ + for (int log2_w = 2; log2_w <= 6; log2_w++) { + for (int log2_h = 0; log2_h <= 6; log2_h++) { + int w = 1 << log2_w; + int h = 1 << log2_h; + ptrdiff_t stride = FFALIGN((w + rnd() % 64), 32); + + if (check_func(c->intra.pred_planar, "vvc_pred_planar_%dx%d_%d", + w, h, bit_depth)) { + randomize_buffers(); + call_ref(buf0, top, left, w, h, stride); + call_new(buf1, top, left, w, h, stride); + if (memcmp(buf0, buf1, BUF_SIZE)) + fail(); + if (w == h) + bench_new(buf1, top, left, w, h, stride); + } + } + } + + report("pred_planar"); +} + +static void check_pred_vh(VVCDSPContext *c, + uint8_t *buf0, uint8_t *buf1, + uint8_t *top, uint8_t *left, int bit_depth) +{ + declare_func(void, uint8_t *src, const uint8_t *ref, + const int w, const int h, const ptrdiff_t stride); + + /* Test all sizes 4x4 up to 64x64, including non-square */ + for (int log2_w = 2; log2_w <= 6; log2_w++) { + for (int log2_h = 2; log2_h <= 6; log2_h++) { + int w = 1 << log2_w; + int h = 1 << log2_h; + ptrdiff_t stride = FFALIGN((w + rnd() % 64), 32); + + if (check_func(c->intra.pred_v, "vvc_pred_v_%dx%d_%d", + w, h, bit_depth)) { + randomize_buffers(); + call_ref(buf0, top, w, h, stride); + call_new(buf1, top, w, h, stride); + if (memcmp(buf0, buf1, BUF_SIZE)) + fail(); + if (w == h) + bench_new(buf1, top, w, h, stride); + } + + if (check_func(c->intra.pred_h, "vvc_pred_h_%dx%d_%d", + w, h, bit_depth)) { + randomize_buffers(); + call_ref(buf0, left, w, h, stride); + call_new(buf1, left, w, h, stride); + if (memcmp(buf0, buf1, BUF_SIZE)) + fail(); + if (w == h) + bench_new(buf1, left, w, h, stride); + } + } + } + + report("pred_vh"); +} + +static void check_pred_mip(VVCDSPContext *c, + uint8_t *buf0, uint8_t *buf1, + uint8_t *top, uint8_t *left, int bit_depth) +{ + declare_func(void, uint8_t *src, const uint8_t *top, + const uint8_t *left, const int w, const int h, + const ptrdiff_t stride, int mode_id, int is_transposed); + + /* Test all sizes 4x4 up to 64x64, including non-square */ + for (int log2_w = 2; log2_w <= 6; log2_w++) { + for (int log2_h = 2; log2_h <= 6; log2_h++) { + int w = 1 << log2_w; + int h = 1 << log2_h; + ptrdiff_t stride = FFALIGN((w + rnd() % 64), 32); + int size_id = ff_vvc_get_mip_size_id(w, h); + + for (int mode_id = 0; mode_id < mip_modes[size_id]; mode_id++) { + if (check_func(c->intra.pred_mip, "vvc_pred_mip_%dx%d_mode%d_%d", + w, h, mode_id, bit_depth)) { + randomize_buffers(); + call_ref(buf0, top, left, w, h, stride, mode_id, 0); + call_new(buf1, top, left, w, h, stride, mode_id, 0); + if (memcmp(buf0, buf1, BUF_SIZE)) + fail(); + + /* Test transposed */ + randomize_buffers(); + call_ref(buf0, top, left, w, h, stride, mode_id, 1); + call_new(buf1, top, left, w, h, stride, mode_id, 1); + if (memcmp(buf0, buf1, BUF_SIZE)) + fail(); + + if (w == h) + bench_new(buf1, top, left, w, h, stride, mode_id, 0); + } + } + } + } + + report("pred_mip"); +} + +static void check_pred_angular(VVCDSPContext *c, + uint8_t *buf0, uint8_t *buf1, + uint8_t *top, uint8_t *left, int bit_depth) +{ + // Set modes to bench + static const int bench_modes[] = { 2, 10, 26, INTRA_DIAG, 42, 58, INTRA_VDIAG }; + + declare_func(void, uint8_t *src, const uint8_t *top, const uint8_t *left, + const int w, const int h, const ptrdiff_t stride, const int c_idx, + const int mode, const int ref_idx, const int filter_flag, + const int need_pdpc); + + /* Test all sizes 4x4 up to 64x64, including non-square */ + for (int log2_w = 2; log2_w <= 6; log2_w++) { + for (int log2_h = 2; log2_h <= 6; log2_h++) { + int w = 1 << log2_w; + int h = 1 << log2_h; + ptrdiff_t stride = FFALIGN((w + rnd() % 64), 32); + + for (int mode = 2; mode <= INTRA_VDIAG; mode++) { + // Skip vertical and horizontal (pred_v, pred_h) + if (mode == INTRA_VERT || mode == INTRA_HORZ) + continue; + + int dir = mode >= INTRA_DIAG; + const char *mode_category; + + if (mode == 2) + mode_category = "Hdiag"; + else if (mode > 2 && mode < INTRA_HORZ) + mode_category = "Hpos"; + else if (mode > INTRA_HORZ && mode < INTRA_DIAG) + mode_category = "Hneg"; + else if (mode == INTRA_DIAG) + mode_category = "Diag"; + else if (mode > INTRA_DIAG && mode < INTRA_VERT) + mode_category = "Vneg"; + else if (mode > INTRA_VERT && mode < INTRA_VDIAG) + mode_category = "Vpos"; + else + mode_category = "Vdiag"; + + for (int ref_idx = 0; ref_idx <= 2; ref_idx++) { + if (check_func(dir ? c->intra.pred_angular_v : c->intra.pred_angular_h, "vvc_pred_angular_%c_%dx%d_%s_mode%d_ref%d_%d", + dir ? 'v' : 'h', w, h, mode_category, mode, ref_idx, bit_depth)) { + int filter_flag = derive_filter_flag(mode, ref_idx, w, h); + int need_pdpc = ff_vvc_need_pdpc(w, h, 0, mode, ref_idx); + + randomize_buffers(); + call_ref(buf0, top, left, w, h, stride, 0, mode, ref_idx, filter_flag, need_pdpc); + call_new(buf1, top, left, w, h, stride, 0, mode, ref_idx, filter_flag, need_pdpc); + if (memcmp(buf0, buf1, BUF_SIZE)) + fail(); + for (int i = 0; i < FF_ARRAY_ELEMS(bench_modes); i++) { + if (mode == bench_modes[i] && w == h && ref_idx == 0) + bench_new(buf1, top, left, w, h, stride, 0, mode, ref_idx, filter_flag, need_pdpc); + } + } + } + // TODO: Add c_idx=1 (chroma) + } + } + } + + report("pred_angular"); +} + +void checkasm_check_vvc_pred(void) +{ + LOCAL_ALIGNED_32(uint8_t, buf0, [BUF_SIZE]); + LOCAL_ALIGNED_32(uint8_t, buf1, [BUF_SIZE]); + LOCAL_ALIGNED_32(uint8_t, top_buf, [REF_BYTES + 16]); + LOCAL_ALIGNED_32(uint8_t, left_buf, [REF_BYTES + 16]); + + for (int bit_depth = 8; bit_depth <= 12; bit_depth += 2) { + VVCDSPContext c; + + uint8_t *top = top_buf + REF_OFFSET * SIZEOF_PIXEL; + uint8_t *left = left_buf + REF_OFFSET * SIZEOF_PIXEL; + + ff_vvc_dsp_init(&c, bit_depth); + check_pred_dc(&c, buf0, buf1, top, left, bit_depth); + check_pred_planar(&c, buf0, buf1, top, left, bit_depth); + check_pred_vh(&c, buf0, buf1, top, left, bit_depth); + check_pred_mip(&c, buf0, buf1, top, left, bit_depth); + check_pred_angular(&c, buf0, buf1, top, left, bit_depth); + } +} diff --git a/tests/fate/checkasm.mak b/tests/fate/checkasm.mak index df05224e76..2da86e218e 100644 --- a/tests/fate/checkasm.mak +++ b/tests/fate/checkasm.mak @@ -97,6 +97,7 @@ FATE_CHECKASM = fate-checkasm-aacencdsp \ fate-checkasm-vp9dsp \ fate-checkasm-vvc_alf \ fate-checkasm-vvc_mc \ + fate-checkasm-vvc_pred \ fate-checkasm-vvc_sao \ $(FATE_CHECKASM): tests/checkasm/checkasm$(EXESUF) -- 2.52.0 >From 73feddaf70d21bf1f9426f84f9da92e708fc1441 Mon Sep 17 00:00:00 2001 From: Marcos Ashton Iglesias <[email protected]> Date: Sun, 13 Sep 2026 23:36:20 +0100 Subject: [PATCH 2/2] avcodec/x86/vvc: add AVX2 pred_planar Widths 4 to 64 at 8 bit, with the height a runtime argument as in the C code. Factoring 2^min(logw,logh) out of the numerator keeps the whole kernel in words: the largest intermediate is 32704, when max(w,h) is 64. The term that is linear in y is carried as a running accumulator, so a row costs one pmullw and three shifts per column group. Widths 32 and 64 run two column groups per pass and write a row with a single 32 byte store. checkasm --bench on a Core Ultra 7 155H: vvc_pred_planar_4x4_8_c: 63.0 vvc_pred_planar_4x4_8_avx2: 20.3 ( 3.10x) vvc_pred_planar_8x8_8_c: 216.7 vvc_pred_planar_8x8_8_avx2: 35.6 ( 6.09x) vvc_pred_planar_16x16_8_c: 966.4 vvc_pred_planar_16x16_8_avx2: 80.3 (12.03x) vvc_pred_planar_32x32_8_c: 3032.7 vvc_pred_planar_32x32_8_avx2: 251.5 (12.06x) vvc_pred_planar_64x64_8_c: 10389.9 vvc_pred_planar_64x64_8_avx2: 967.1 (10.74x) --- libavcodec/x86/vvc/Makefile | 1 + libavcodec/x86/vvc/dsp_init.c | 42 ++++++++ libavcodec/x86/vvc/intra.asm | 189 ++++++++++++++++++++++++++++++++++ 3 files changed, 232 insertions(+) create mode 100644 libavcodec/x86/vvc/intra.asm diff --git a/libavcodec/x86/vvc/Makefile b/libavcodec/x86/vvc/Makefile index 0cebfb4e9e..b24ea6e692 100644 --- a/libavcodec/x86/vvc/Makefile +++ b/libavcodec/x86/vvc/Makefile @@ -4,6 +4,7 @@ clean:: X86ASM-OBJS-$(CONFIG_VVC_DECODER) += x86/vvc/dsp_init.o \ x86/vvc/alf.o \ x86/vvc/dmvr.o \ + x86/vvc/intra.o \ x86/vvc/mc.o \ x86/vvc/of.o \ x86/vvc/sad.o \ diff --git a/libavcodec/x86/vvc/dsp_init.c b/libavcodec/x86/vvc/dsp_init.c index 6802294795..3ef60c6607 100644 --- a/libavcodec/x86/vvc/dsp_init.c +++ b/libavcodec/x86/vvc/dsp_init.c @@ -24,6 +24,7 @@ #include "config.h" #include "libavutil/attributes.h" +#include "libavutil/avassert.h" #include "libavutil/cpu.h" #include "libavutil/x86/cpu.h" #include "libavcodec/vvc/dec.h" @@ -294,6 +295,44 @@ void bf(ff_vvc_alf_filter_chroma, bd, opt)(uint8_t *dst, ptrdiff_t dst_stride, \ #endif +#if HAVE_AVX2_EXTERNAL +#define PLANAR_PROTOTYPE(wd, bd, opt) \ +void ff_vvc_pred_planar_w##wd##_##bd##_##opt(uint8_t *src, const uint8_t *top, \ + const uint8_t *left, int w, int h, ptrdiff_t stride); + +#define PLANAR_PROTOTYPES(bd, opt) \ + PLANAR_PROTOTYPE( 4, bd, opt) \ + PLANAR_PROTOTYPE( 8, bd, opt) \ + PLANAR_PROTOTYPE(16, bd, opt) \ + PLANAR_PROTOTYPE(32, bd, opt) \ + PLANAR_PROTOTYPE(64, bd, opt) + +PLANAR_PROTOTYPES(8, avx2) + +/* pred_planar is one pointer taking a runtime width, so dispatch here. */ +#define PLANAR_DISPATCH(bd, opt) \ +static void vvc_pred_planar_##bd##_##opt(uint8_t *src, const uint8_t *top, \ + const uint8_t *left, const int w, const int h, const ptrdiff_t stride) \ +{ \ + static void (*const planar[])(uint8_t *src, const uint8_t *top, \ + const uint8_t *left, int w, int h, ptrdiff_t stride) = { \ + ff_vvc_pred_planar_w4_##bd##_##opt, \ + ff_vvc_pred_planar_w8_##bd##_##opt, \ + ff_vvc_pred_planar_w16_##bd##_##opt, \ + ff_vvc_pred_planar_w32_##bd##_##opt, \ + ff_vvc_pred_planar_w64_##bd##_##opt, \ + }; \ + av_assert2(w >= 4 && w <= 64 && !(w & (w - 1))); \ + planar[av_log2(w) - 2](src, top, left, w, h, stride); \ +} + +PLANAR_DISPATCH(8, avx2) + +#define INTRA_INIT(bd, opt) do { \ + c->intra.pred_planar = vvc_pred_planar_##bd##_##opt; \ +} while (0) +#endif + #endif // ARCH_X86_64 @@ -318,6 +357,9 @@ av_cold void ff_vvc_dsp_init_x86(VVCDSPContext *const c, const int bd) OF_INIT(8, avx2); SAD_INIT(); + // intra + INTRA_INIT(8, avx2); + // filter ALF_INIT(8, avx2); SAO_INIT(8, avx2); diff --git a/libavcodec/x86/vvc/intra.asm b/libavcodec/x86/vvc/intra.asm new file mode 100644 index 0000000000..f463dd272c --- /dev/null +++ b/libavcodec/x86/vvc/intra.asm @@ -0,0 +1,189 @@ +;****************************************************************************** +;* SIMD-optimized VVC intra prediction +;* +;* This file is part of FFmpeg. +;* +;* FFmpeg is free software; you can redistribute it and/or +;* modify it under the terms of the GNU Lesser General Public +;* License as published by the Free Software Foundation; either +;* version 2.1 of the License, or (at your option) any later version. +;* +;* FFmpeg is distributed in the hope that it will be useful, +;* but WITHOUT ANY WARRANTY; without even the implied warranty of +;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;* Lesser General Public License for more details. +;* +;* You should have received a copy of the GNU Lesser General Public +;* License along with FFmpeg; if not, write to the Free Software +;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +;****************************************************************************** + +%include "libavutil/x86/x86util.asm" + +%if ARCH_X86_64 +%if HAVE_AVX2_EXTERNAL + +SECTION_RODATA 32 + +; (w - 1 - x) is a suffix of pw_desc, (x + 1) a prefix of pw_asc. The tail keeps +; the w = 4 group, which is read eight words wide, inside the table. +pw_desc: dw 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48 + dw 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32 + dw 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16 + dw 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 + dw 0, 0, 0, 0, 0, 0, 0, 0 +pw_asc: dw 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + dw 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 + dw 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48 + dw 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64 + +cextern pw_255 + +SECTION .text + +;----------------------------------------------------------------------------- +; void ff_vvc_pred_planar_w<w>_8_avx2(uint8_t *src, const uint8_t *top, +; const uint8_t *left, int w, int h, +; ptrdiff_t stride) +;----------------------------------------------------------------------------- + +; src[y*stride + x] = (A << logw) + (B << logh) + w*h >> (logw + logh + 1) +; A = (h-1-y)*top[x] + (y+1)*left[h] +; B = (w-1-x)*left[y] + (x+1)*top[w] +; +; Factoring 2^min(logw,logh) out of the numerator gives +; pred = ((A << sa) + (B << sb) + max(w,h)) >> (sc + 1) +; with sc = max(logw,logh), sa = logw - min, sb = logh - min, so one shift is +; always zero. (A << sa) and (B << sb) are each at most 255*max(w,h), so the +; numerator peaks at 511*max(w,h) = 32704 and the kernel stays in words. A is +; linear in y with the loop-invariant step left[h] - top[x], so it is carried +; as a running accumulator and a row costs one pmullw for B plus three shifts. + +; Set up the running accumulator A(0) and its per-row step for the columns +; starting at %4. Clobbers m6 and m7. +; %1 = accumulator, %2 = step, %3 = (x+1)*top[w], %4 = first column, %5 = width +%macro PLANAR_INIT 5 + vpbroadcastb %3, [topq + %5] + pand %3, m5 ; top[w] + pmullw %3, [pw_asc + (%4) * 2] ; (x+1)*top[w] + vpbroadcastb %1, [leftq + hq] + pand %1, m5 ; left[h] + pmovzxbw m7, [topq + %4] ; top[x] + psubw %2, %1, m7 ; step = left[h] - top[x] + lea tmpd, [hq - 1] + movd xm6, tmpd + vpbroadcastw m6, xm6 + pmullw m6, m7 + paddw %1, m6 ; A(0) +%endmacro + +; One row of one group: %1 = A, %2 = (x+1)*top[w], %3 = left[y] (preserved), +; %4 = first column, %5 = width, %6 = output +%macro PLANAR_ROW 6 + pmullw m6, %3, [pw_desc + (64 - %5 + %4) * 2] + paddw m6, %2 ; B + psllw m6, xm9 ; B << sb + psllw %6, %1, xm8 ; A << sa + paddw %6, m6 + paddw %6, m4 + psrlw %6, xm10 +%endmacro + +; One group of mmsize/2 columns, for widths of at most that. +; %1 = width, %2 = first column +%macro PLANAR_GROUP 2 + PLANAR_INIT m0, m1, m2, %2, %1 + lea srcq, [baseq + %2] + xor yd, yd +%%loop: + vpbroadcastb m3, [leftq + yq] + pand m3, m5 ; left[y] + PLANAR_ROW m0, m2, m3, %2, %1, m7 + packuswb m7, m7 +%if %1 == 4 + movd [srcq], xm7 +%elif %1 == 8 + movq [srcq], xm7 +%else + vpermq m7, m7, q3120 + movu [srcq], xm7 +%endif + add srcq, strideq + paddw m0, m1 + inc yd + cmp yd, hd + jl %%loop +%endmacro + +; Two adjacent groups of 16 columns per pass, so that the left[y] broadcast +; is shared and a row is written with a single 32 byte store. +; %1 = width, %2 = first column +%macro PLANAR_GROUP2 2 + PLANAR_INIT m0, m1, m2, %2, %1 + PLANAR_INIT m11, m12, m13, %2 + 16, %1 + lea srcq, [baseq + %2] + xor yd, yd +%%loop: + vpbroadcastb m3, [leftq + yq] + pand m3, m5 ; left[y] + PLANAR_ROW m0, m2, m3, %2, %1, m7 + PLANAR_ROW m11, m13, m3, %2 + 16, %1, m14 + packuswb m7, m14 + vpermq m7, m7, q3120 + movu [srcq], m7 + add srcq, strideq + paddw m0, m1 + paddw m11, m12 + inc yd + cmp yd, hd + jl %%loop +%endmacro + +; %1 = width, %2 = log2(width) +%macro PRED_PLANAR 2 +cglobal vvc_pred_planar_w%1_8, 6, 11, 15, src, top, left, w, h, stride, base, y, tmp, sha, shb + movsxdifnidn hq, hd + bsr tmpd, hd ; logh + mov shad, %2 + sub shad, tmpd ; logw - logh + mov shbd, tmpd + sub shbd, %2 ; logh - logw + xor tmpd, tmpd + test shad, shad + cmovs shad, tmpd + test shbd, shbd + cmovs shbd, tmpd + movd xm8, shad + movd xm9, shbd + mov tmpd, %1 + cmp tmpd, hd + cmovl tmpd, hd ; max(w,h) = 1 << sc + movd xm4, tmpd + vpbroadcastw m4, xm4 + bsr tmpd, tmpd + inc tmpd + movd xm10, tmpd ; sc + 1 + mova m5, [pw_255] + mov baseq, srcq +%if %1 <= 16 + PLANAR_GROUP %1, 0 +%else +%assign %%c 0 +%rep %1 / 32 + PLANAR_GROUP2 %1, %%c +%assign %%c %%c + 32 +%endrep +%endif + RET +%endmacro + +INIT_XMM avx2 +PRED_PLANAR 4, 2 +PRED_PLANAR 8, 3 +INIT_YMM avx2 +PRED_PLANAR 16, 4 +PRED_PLANAR 32, 5 +PRED_PLANAR 64, 6 + +%endif ; HAVE_AVX2_EXTERNAL +%endif ; ARCH_X86_64 -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
