PR #20899 opened by Zhao Zhili (quink) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20899 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/20899.patch
avutil/hwcontext_vaapi: fix use fourcc not supported by devices 1. A AVPixelFormat can map to multiple VA_FOURCCs, while vaapi_format_from_pix_fmt() only returns the first item matched before this patch. 2. vaapi_frames_init() use vaapi_format_from_pix_fmt() to get the first item. fourcc in this item may not be supported by the device. There is a real issue. av_hwdevice_get_hwframe_constraints returned AV_PIX_FMT_YUV422P is a supported sw_format, while it's only supported by VA_FOURCC_YV16, but vaapi_frames_init try to use VA_FOURCC_422H later. This patch makes vaapi_format_from_pix_fmt return all matched items iteratively, then use strict check in vaapi_frames_init to get the right fourcc. >From 6bb427af54bf5453b998c3c12a389b087217aeda Mon Sep 17 00:00:00 2001 From: Zhao Zhili <[email protected]> Date: Wed, 12 Nov 2025 21:56:36 +0800 Subject: [PATCH 1/2] avutil/hwcontext_vaapi: fix error code for invalid argument Also replace goto by return directly, as there is nothing to cleanup. --- libavutil/hwcontext_vaapi.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c index 753dcf8905..256363eb97 100644 --- a/libavutil/hwcontext_vaapi.c +++ b/libavutil/hwcontext_vaapi.c @@ -222,9 +222,12 @@ static int vaapi_get_image_format(AVHWDeviceContext *hwdev, const VAAPIFormatDescriptor *desc; int i; + if (!image_format) + return AVERROR(EINVAL); + desc = vaapi_format_from_pix_fmt(pix_fmt); - if (!desc || !image_format) - goto fail; + if (!desc) + return AVERROR(ENOSYS); for (i = 0; i < ctx->nb_formats; i++) { if (ctx->formats[i].fourcc == desc->fourcc) { @@ -233,7 +236,6 @@ static int vaapi_get_image_format(AVHWDeviceContext *hwdev, } } -fail: return AVERROR(ENOSYS); } -- 2.49.1 >From b71c350f9f176dab871c2a89a60a7c58909c81f4 Mon Sep 17 00:00:00 2001 From: Zhao Zhili <[email protected]> Date: Wed, 12 Nov 2025 22:35:45 +0800 Subject: [PATCH 2/2] avutil/hwcontext_vaapi: fix use fourcc not supported by devices 1. A AVPixelFormat can map to multiple VA_FOURCCs, while vaapi_format_from_pix_fmt() only returns the first item matched before this patch. 2. vaapi_frames_init() use vaapi_format_from_pix_fmt() to get the first item. fourcc in this item may not be supported by the device. There is a real issue. av_hwdevice_get_hwframe_constraints returned AV_PIX_FMT_YUV422P is a supported sw_format, while it's only supported by VA_FOURCC_YV16, but vaapi_frames_init try to use VA_FOURCC_422H later. This patch makes vaapi_format_from_pix_fmt return all matched items iteratively, then use strict check in vaapi_frames_init to get the right fourcc. --- libavutil/hwcontext_vaapi.c | 72 ++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c index 256363eb97..4f3502797b 100644 --- a/libavutil/hwcontext_vaapi.c +++ b/libavutil/hwcontext_vaapi.c @@ -195,12 +195,17 @@ static const VAAPIFormatDescriptor * } static const VAAPIFormatDescriptor * - vaapi_format_from_pix_fmt(enum AVPixelFormat pix_fmt) + vaapi_format_from_pix_fmt(enum AVPixelFormat pix_fmt, const VAAPIFormatDescriptor *prev) { - int i; - for (i = 0; i < FF_ARRAY_ELEMS(vaapi_format_map); i++) - if (vaapi_format_map[i].pix_fmt == pix_fmt) - return &vaapi_format_map[i]; + const VAAPIFormatDescriptor *end = &vaapi_format_map[FF_ARRAY_ELEMS(vaapi_format_map)]; + if (!prev) + prev = vaapi_format_map; + else + prev++; + + for (; prev < end; prev++) + if (prev->pix_fmt == pix_fmt) + return prev; return NULL; } @@ -214,31 +219,39 @@ static enum AVPixelFormat vaapi_pix_fmt_from_fourcc(unsigned int fourcc) return AV_PIX_FMT_NONE; } -static int vaapi_get_image_format(AVHWDeviceContext *hwdev, +static int vaapi_get_img_desc_and_format(AVHWDeviceContext *hwdev, enum AVPixelFormat pix_fmt, + const VAAPIFormatDescriptor **_desc, VAImageFormat **image_format) { VAAPIDeviceContext *ctx = hwdev->hwctx; - const VAAPIFormatDescriptor *desc; + const VAAPIFormatDescriptor *desc = NULL; int i; - if (!image_format) - return AVERROR(EINVAL); - - desc = vaapi_format_from_pix_fmt(pix_fmt); - if (!desc) - return AVERROR(ENOSYS); - - for (i = 0; i < ctx->nb_formats; i++) { - if (ctx->formats[i].fourcc == desc->fourcc) { - *image_format = &ctx->formats[i].image_format; - return 0; + while ((desc = vaapi_format_from_pix_fmt(pix_fmt, desc))) { + for (i = 0; i < ctx->nb_formats; i++) { + if (ctx->formats[i].fourcc == desc->fourcc) { + if (_desc) + *_desc = desc; + if (image_format) + *image_format = &ctx->formats[i].image_format; + return 0; + } } } return AVERROR(ENOSYS); } +static int vaapi_get_image_format(AVHWDeviceContext *hwdev, + enum AVPixelFormat pix_fmt, + VAImageFormat **image_format) +{ + if (!image_format) + return AVERROR(EINVAL); + return vaapi_get_img_desc_and_format(hwdev, pix_fmt, NULL, image_format); +} + static int vaapi_frames_get_constraints(AVHWDeviceContext *hwdev, const void *hwconfig, AVHWFramesConstraints *constraints) @@ -564,19 +577,23 @@ static int vaapi_frames_init(AVHWFramesContext *hwfc) VAAPIFramesContext *ctx = hwfc->hwctx; AVVAAPIFramesContext *avfc = &ctx->p; AVVAAPIDeviceContext *hwctx = hwfc->device_ctx->hwctx; - const VAAPIFormatDescriptor *desc; - VAImageFormat *expected_format; + const VAAPIFormatDescriptor *desc = NULL; + VAImageFormat *expected_format = NULL; AVBufferRef *test_surface = NULL; VASurfaceID test_surface_id; VAImage test_image; VAStatus vas; int err, i; - desc = vaapi_format_from_pix_fmt(hwfc->sw_format); - if (!desc) { - av_log(hwfc, AV_LOG_ERROR, "Unsupported format: %s.\n", - av_get_pix_fmt_name(hwfc->sw_format)); - return AVERROR(EINVAL); + err = vaapi_get_img_desc_and_format(hwfc->device_ctx, hwfc->sw_format, + &desc, &expected_format); + if (err < 0) { + // Use a relaxed check when pool exist. It can be an external pool. + if (!hwfc->pool || !vaapi_format_from_pix_fmt(hwfc->sw_format, NULL)) { + av_log(hwfc, AV_LOG_ERROR, "Unsupported format: %s.\n", + av_get_pix_fmt_name(hwfc->sw_format)); + return AVERROR(EINVAL); + } } if (!hwfc->pool) { @@ -675,10 +692,7 @@ static int vaapi_frames_init(AVHWFramesContext *hwfc) test_surface_id = (VASurfaceID)(uintptr_t)test_surface->data; ctx->derive_works = 0; - - err = vaapi_get_image_format(hwfc->device_ctx, - hwfc->sw_format, &expected_format); - if (err == 0) { + if (expected_format) { vas = vaDeriveImage(hwctx->display, test_surface_id, &test_image); if (vas == VA_STATUS_SUCCESS) { if (expected_format->fourcc == test_image.format.fourcc) { -- 2.49.1 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
