[FFmpeg-devel] [PATCH 1/2] avcodec/vvcdec: deblock, fix uninitialized values

[Nuo Mi]

see 
https://fate.ffmpeg.org/report.cgi?slot=x86_64-archlinux-gcc-valgrind&time=20240105201935

If tc is zero, the max_len_q, max_len_p are uninitialized.

Reported-by: James Almer 
---
 libavcodec/vvc/vvc_filter_template.c | 231 +--
 1 file changed, 114 insertions(+), 117 deletions(-)

diff --git a/libavcodec/vvc/vvc_filter_template.c 
b/libavcodec/vvc/vvc_filter_template.c
index a4f1792ec4..e30deaac96 100644
--- a/libavcodec/vvc/vvc_filter_template.c
+++ b/libavcodec/vvc/vvc_filter_template.c
@@ -798,81 +798,78 @@ static void FUNC(vvc_loop_filter_luma)(uint8_t* _pix, 
ptrdiff_t _xstride, ptrdif
 const ptrdiff_t ystride = _ystride / sizeof(pixel);
 
 for (int i = 0; i < 2; i++) {
-pixel* pix  = (pixel*)_pix + i * 4 * ystride;
-const int dp0   = abs(P2 - 2 * P1 + P0);
-const int dq0   = abs(Q2 - 2 * Q1 + Q0);
-const int dp3   = abs(TP2 - 2 * TP1 + TP0);
-const int dq3   = abs(TQ2 - 2 * TQ1 + TQ0);
-const int d0= dp0 + dq0;
-const int d3= dp3 + dq3;
 #if BIT_DEPTH < 10
 const int tc= (_tc[i] + (1 << (9 - BIT_DEPTH))) >> (10 - 
BIT_DEPTH);
 #else
 const int tc= _tc[i] << (BIT_DEPTH - 10);
 #endif
-const int tc25  = ((tc * 5 + 1) >> 1);
-
-const int no_p  = _no_p[i];
-const int no_q  = _no_q[i];
-
-int max_len_p   = _max_len_p[i];
-int max_len_q   = _max_len_q[i];
-
-const int large_p = (max_len_p > 3 && !hor_ctu_edge);
-const int large_q = max_len_q > 3;
-const int beta = _beta[i] << BIT_DEPTH - 8;
-
-const int beta_3 = beta >> 3;
-const int beta_2 = beta >> 2;
-
-if (!tc)
-continue;
-
-if (large_p || large_q) {
-const int dp0l = large_p ? ((dp0 + abs(P5 - 2 * P4 + P3) + 1) >> 
1) : dp0;
-const int dq0l = large_q ? ((dq0 + abs(Q5 - 2 * Q4 + Q3) + 1) >> 
1) : dq0;
-const int dp3l = large_p ? ((dp3 + abs(TP5 - 2 * TP4 + TP3) + 1) 
>> 1) : dp3;
-const int dq3l = large_q ? ((dq3 + abs(TQ5 - 2 * TQ4 + TQ3) + 1) 
>> 1) : dq3;
-const int d0l = dp0l + dq0l;
-const int d3l = dp3l + dq3l;
-const int beta53 = beta * 3 >> 5;
-const int beta_4 = beta >> 4;
-max_len_p = large_p ? max_len_p : 3;
-max_len_q = large_q ? max_len_q : 3;
-
-if (d0l + d3l < beta) {
-const int sp0l = abs(P3 - P0) + (max_len_p == 7 ? abs(P7 - P6 
- P5 + P4) : 0);
-const int sq0l = abs(Q0 - Q3) + (max_len_q == 7 ? abs(Q4 - Q5 
- Q6 + Q7) : 0);
-const int sp3l = abs(TP3 - TP0) + (max_len_p == 7 ? abs(TP7 - 
TP6 - TP5 + TP4) : 0);
-const int sq3l = abs(TQ0 - TQ3) + (max_len_q == 7 ? abs(TQ4 - 
TQ5 - TQ6 + TQ7) : 0);
-const int sp0 = large_p ? ((sp0l + abs(P3 -   P(max_len_p)) + 
1) >> 1) : sp0l;
-const int sp3 = large_p ? ((sp3l + abs(TP3 - TP(max_len_p)) + 
1) >> 1) : sp3l;
-const int sq0 = large_q ? ((sq0l + abs(Q3 -   Q(max_len_q)) + 
1) >> 1) : sq0l;
-const int sq3 = large_q ? ((sq3l + abs(TQ3 - TQ(max_len_q)) + 
1) >> 1) : sq3l;
-if (sp0 + sq0 < beta53 && abs(P0 - Q0) < tc25 &&
-sp3 + sq3 < beta53 && abs(TP0 - TQ0) < tc25 &&
-(d0l << 1) < beta_4 && (d3l << 1) < beta_4) {
-FUNC(loop_filter_luma_large)(pix, xstride, ystride, tc, 
no_p, no_q, max_len_p, max_len_q);
-continue;
+if (tc) {
+pixel* pix = (pixel*)_pix + i * 4 * ystride;
+const int dp0  = abs(P2 - 2 * P1 + P0);
+const int dq0  = abs(Q2 - 2 * Q1 + Q0);
+const int dp3  = abs(TP2 - 2 * TP1 + TP0);
+const int dq3  = abs(TQ2 - 2 * TQ1 + TQ0);
+const int d0   = dp0 + dq0;
+const int d3   = dp3 + dq3;
+const int tc25 = ((tc * 5 + 1) >> 1);
+const int no_p = _no_p[i];
+const int no_q = _no_q[i];
+
+int max_len_p = _max_len_p[i];
+int max_len_q = _max_len_q[i];
+const int large_p = (max_len_p > 3 && !hor_ctu_edge);
+const int large_q = max_len_q > 3;
+
+const int beta   = _beta[i] << BIT_DEPTH - 8;
+const int beta_3 = beta >> 3;
+const int beta_2 = beta >> 2;
+
+if (large_p || large_q) {
+const int dp0l = large_p ? ((dp0 + abs(P5 - 2 * P4 + P3) + 1) 
>> 1) : dp0;
+const int dq0l = large_q ? ((dq0 + abs(Q5 - 2 * Q4 + Q3) + 1) 
>> 1) : dq0;
+const int dp3l = large_p ? ((dp3 + abs(TP5 - 2 * TP4 + TP3) + 
1) >> 1) : dp3;
+const int dq3l = large_q ? ((dq3 + abs(TQ5 - 2 * TQ4 + TQ3) + 
1) >> 1) : dq3;
+const int d0l  = dp0l + dq0l;
+const int d3l  = dp3l + dq3l;
+const int beta53 = be

Re: [FFmpeg-devel] [PATCH 1/2] avcodec/vvcdec: deblock, fix uninitialized values

[Nuo Mi]

On Sun, Jan 7, 2024 at 12:43 PM Nuo Mi  wrote:

> see
> https://fate.ffmpeg.org/report.cgi?slot=x86_64-archlinux-gcc-valgrind&time=20240105201935
>
> If tc is zero, the max_len_q, max_len_p are uninitialized.
>
> Reported-by: James Almer 
>
Hi James,
Could you help try and review this?
Thank you

> ---
>  libavcodec/vvc/vvc_filter_template.c | 231 +--
>  1 file changed, 114 insertions(+), 117 deletions(-)
>
> diff --git a/libavcodec/vvc/vvc_filter_template.c
> b/libavcodec/vvc/vvc_filter_template.c
> index a4f1792ec4..e30deaac96 100644
> --- a/libavcodec/vvc/vvc_filter_template.c
> +++ b/libavcodec/vvc/vvc_filter_template.c
> @@ -798,81 +798,78 @@ static void FUNC(vvc_loop_filter_luma)(uint8_t*
> _pix, ptrdiff_t _xstride, ptrdif
>  const ptrdiff_t ystride = _ystride / sizeof(pixel);
>
>  for (int i = 0; i < 2; i++) {
> -pixel* pix  = (pixel*)_pix + i * 4 * ystride;
> -const int dp0   = abs(P2 - 2 * P1 + P0);
> -const int dq0   = abs(Q2 - 2 * Q1 + Q0);
> -const int dp3   = abs(TP2 - 2 * TP1 + TP0);
> -const int dq3   = abs(TQ2 - 2 * TQ1 + TQ0);
> -const int d0= dp0 + dq0;
> -const int d3= dp3 + dq3;
>  #if BIT_DEPTH < 10
>  const int tc= (_tc[i] + (1 << (9 - BIT_DEPTH))) >> (10 -
> BIT_DEPTH);
>  #else
>  const int tc= _tc[i] << (BIT_DEPTH - 10);
>  #endif
> -const int tc25  = ((tc * 5 + 1) >> 1);
> -
> -const int no_p  = _no_p[i];
> -const int no_q  = _no_q[i];
> -
> -int max_len_p   = _max_len_p[i];
> -int max_len_q   = _max_len_q[i];
> -
> -const int large_p = (max_len_p > 3 && !hor_ctu_edge);
> -const int large_q = max_len_q > 3;
> -const int beta = _beta[i] << BIT_DEPTH - 8;
> -
> -const int beta_3 = beta >> 3;
> -const int beta_2 = beta >> 2;
> -
> -if (!tc)
> -continue;
> -
> -if (large_p || large_q) {
> -const int dp0l = large_p ? ((dp0 + abs(P5 - 2 * P4 + P3) + 1)
> >> 1) : dp0;
> -const int dq0l = large_q ? ((dq0 + abs(Q5 - 2 * Q4 + Q3) + 1)
> >> 1) : dq0;
> -const int dp3l = large_p ? ((dp3 + abs(TP5 - 2 * TP4 + TP3) +
> 1) >> 1) : dp3;
> -const int dq3l = large_q ? ((dq3 + abs(TQ5 - 2 * TQ4 + TQ3) +
> 1) >> 1) : dq3;
> -const int d0l = dp0l + dq0l;
> -const int d3l = dp3l + dq3l;
> -const int beta53 = beta * 3 >> 5;
> -const int beta_4 = beta >> 4;
> -max_len_p = large_p ? max_len_p : 3;
> -max_len_q = large_q ? max_len_q : 3;
> -
> -if (d0l + d3l < beta) {
> -const int sp0l = abs(P3 - P0) + (max_len_p == 7 ? abs(P7
> - P6 - P5 + P4) : 0);
> -const int sq0l = abs(Q0 - Q3) + (max_len_q == 7 ? abs(Q4
> - Q5 - Q6 + Q7) : 0);
> -const int sp3l = abs(TP3 - TP0) + (max_len_p == 7 ?
> abs(TP7 - TP6 - TP5 + TP4) : 0);
> -const int sq3l = abs(TQ0 - TQ3) + (max_len_q == 7 ?
> abs(TQ4 - TQ5 - TQ6 + TQ7) : 0);
> -const int sp0 = large_p ? ((sp0l + abs(P3 -
>  P(max_len_p)) + 1) >> 1) : sp0l;
> -const int sp3 = large_p ? ((sp3l + abs(TP3 -
> TP(max_len_p)) + 1) >> 1) : sp3l;
> -const int sq0 = large_q ? ((sq0l + abs(Q3 -
>  Q(max_len_q)) + 1) >> 1) : sq0l;
> -const int sq3 = large_q ? ((sq3l + abs(TQ3 -
> TQ(max_len_q)) + 1) >> 1) : sq3l;
> -if (sp0 + sq0 < beta53 && abs(P0 - Q0) < tc25 &&
> -sp3 + sq3 < beta53 && abs(TP0 - TQ0) < tc25 &&
> -(d0l << 1) < beta_4 && (d3l << 1) < beta_4) {
> -FUNC(loop_filter_luma_large)(pix, xstride, ystride,
> tc, no_p, no_q, max_len_p, max_len_q);
> -continue;
> +if (tc) {
> +pixel* pix = (pixel*)_pix + i * 4 * ystride;
> +const int dp0  = abs(P2 - 2 * P1 + P0);
> +const int dq0  = abs(Q2 - 2 * Q1 + Q0);
> +const int dp3  = abs(TP2 - 2 * TP1 + TP0);
> +const int dq3  = abs(TQ2 - 2 * TQ1 + TQ0);
> +const int d0   = dp0 + dq0;
> +const int d3   = dp3 + dq3;
> +const int tc25 = ((tc * 5 + 1) >> 1);
> +const int no_p = _no_p[i];
> +const int no_q = _no_q[i];
> +
> +int max_len_p = _max_len_p[i];
> +int max_len_q = _max_len_q[i];
> +const int large_p = (max_len_p > 3 && !hor_ctu_edge);
> +const int large_q = max_len_q > 3;
> +
> +const int beta   = _beta[i] << BIT_DEPTH - 8;
> +const int beta_3 = beta >> 3;
> +const int beta_2 = beta >> 2;
> +
> +if (large_p || large_q) {
> +const int dp0l = large_p ? ((dp0 + abs(P5 - 2 * P4 + P3)
> + 1) >> 1) : dp0;
> +const int dq0l = large_q ? ((dq0 + abs(Q5 - 2 * Q4 + Q3)
> + 1) >> 1) : dq0;
> +

Re: [FFmpeg-devel] [PATCH 1/2] avcodec/vvcdec: deblock, fix uninitialized values

[James Almer]

On 1/8/2024 10:00 PM, Nuo Mi wrote:



On Sun, Jan 7, 2024 at 12:43 PM Nuo Mi > wrote:


see

https://fate.ffmpeg.org/report.cgi?slot=x86_64-archlinux-gcc-valgrind&time=20240105201935 


If tc is zero, the max_len_q, max_len_p are uninitialized.

Reported-by: James Almer mailto:jamr...@gmail.com>>

Hi James,
Could you help try and review this?


The patch doesn't seem to apply.
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] avcodec/vvcdec: deblock, fix uninitialized values

[Nuo Mi]

On Tue, Jan 16, 2024 at 10:38 PM James Almer  wrote:

> On 1/8/2024 10:00 PM, Nuo Mi wrote:
> >
> >
> > On Sun, Jan 7, 2024 at 12:43 PM Nuo Mi  > > wrote:
> >
> > see
> >
> https://fate.ffmpeg.org/report.cgi?slot=x86_64-archlinux-gcc-valgrind&time=20240105201935
> <
> https://fate.ffmpeg.org/report.cgi?slot=x86_64-archlinux-gcc-valgrind&time=20240105201935
> >
> >
> > If tc is zero, the max_len_q, max_len_p are uninitialized.
> >
> > Reported-by: James Almer  jamr...@gmail.com>>
> >
> > Hi James,
> > Could you help try and review this?
>
> The patch doesn't seem to apply.
>
Sorry, the master changed.
Please use the v2.
Thank you

> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
>
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".