PR #24951 opened by t-boiko URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24951 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24951.patch
vulkan_decode: copy layered DPB out when COINCIDE-only without separate refs 54e7ef195e stopped failing init on COINCIDE without SEPARATE_REFERENCE_IMAGES and kept using the output image as a DPB slot. That is not a conforming coincide decode: without separate refs every DPB slot is a layer of one VkImage (VUID-VkVideoBeginCodingInfoKHR-flags-07244), and the destination must match the reconstructed picture (VUID-07141). Decode into that layered DPB, then vkCmdCopyImage the layer into a normal non-layered AVFrame. Returning the DPB array layer as the frame would break filters, hwmap, and DRM export, which expect one image per frame. Skip the copy when AV1 film grain is applied. Grain already requires the destination to differ from the reconstructed picture (VUID-09249). NVIDIA Ampere reports this capability set for every decode profile. >From c77e2679041fd4a659e247b82987ea1f74ee12f0 Mon Sep 17 00:00:00 2001 From: Tymur Boiko <[email protected]> Date: Mon, 5 Oct 2026 02:18:08 +0200 Subject: [PATCH] vulkan_decode: copy layered DPB out when COINCIDE-only without separate refs Those profiles cannot bind a distinct user frame as decode dst. Decode into the layered DPB and copy to the AVFrame so FFmpeg still sees one image per frame. Skip the copy for AV1 film grain, which already requires dst != DPB. --- libavcodec/vulkan_decode.c | 192 ++++++++++++++++++++++++++++++------- 1 file changed, 157 insertions(+), 35 deletions(-) diff --git a/libavcodec/vulkan_decode.c b/libavcodec/vulkan_decode.c index 34ca4dc565..7d4af7699f 100644 --- a/libavcodec/vulkan_decode.c +++ b/libavcodec/vulkan_decode.c @@ -23,6 +23,7 @@ #include "config_components.h" #include "libavutil/avassert.h" #include "libavutil/mem.h" +#include "libavutil/pixdesc.h" #include "libavutil/vulkan_loader.h" #include "h264dec.h" @@ -421,6 +422,44 @@ static int decode_reset(AVCodecContext *avctx, FFVulkanDecodeShared *ctx) return err; } +static int vk_decode_copy_out(FFVulkanDecodeShared *ctx) +{ + return ctx->common.layered_dpb && + !(ctx->dec_caps.flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR); +} + +static void vk_decode_copy_layer(FFVulkanFunctions *vk, VkCommandBuffer cmd, + VkImage src, uint32_t layer, AVFrame *dst, + VkExtent2D ext) +{ + AVVkFrame *vkf = (AVVkFrame *)dst->data[0]; + AVHWFramesContext *hwfc = (AVHWFramesContext *)dst->hw_frames_ctx->data; + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format); + const int nb_planes = av_pix_fmt_count_planes(hwfc->sw_format); + const int nb_images = ff_vk_count_images(vkf); + + for (int i = 0; i < nb_planes; i++) { + /* AV_CEIL_RSHIFT is signed; VkExtent2D width/height are uint32_t. */ + int w = ext.width, h = ext.height; + VkImageAspectFlags aspect = ff_vk_aspect_flag(dst, i); + VkImageCopy r; + + if (i && desc) { + w = AV_CEIL_RSHIFT(w, desc->log2_chroma_w); + h = AV_CEIL_RSHIFT(h, desc->log2_chroma_h); + } + r = (VkImageCopy) { + .srcSubresource = { .aspectMask = aspect, .baseArrayLayer = layer, + .layerCount = 1 }, + .dstSubresource = { .aspectMask = aspect, .layerCount = 1 }, + .extent = { w, h, 1 }, + }; + vk->CmdCopyImage(cmd, src, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + vkf->img[FFMIN(i, nb_images - 1)], + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &r); + } +} + int ff_vk_decode_frame(AVCodecContext *avctx, AVFrame *pic, FFVulkanDecodePicture *vp, AVFrame *rpic[], FFVulkanDecodePicture *rvkp[]) @@ -439,6 +478,17 @@ int ff_vk_decode_frame(AVCodecContext *avctx, /* Quirks */ const int layered_dpb = ctx->common.layered_dpb; + int copy = vk_decode_copy_out(ctx); + +#if CONFIG_AV1_VULKAN_HWACCEL + if (copy) { + const VkVideoDecodeAV1PictureInfoKHR *a = + ff_vk_find_struct(vp->decode_info.pNext, + VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PICTURE_INFO_KHR); + if (a && a->pStdPictureInfo && a->pStdPictureInfo->flags.apply_grain) + copy = 0; + } +#endif VkVideoBeginCodingInfoKHR decode_start = { .sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR, @@ -512,8 +562,10 @@ int ff_vk_decode_frame(AVCodecContext *avctx, ff_vk_exec_add_dep_refstruct(&ctx->s, exec, dec->session_params); err = ff_vk_exec_add_dep_frame(&ctx->s, exec, pic, - VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR, - VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR); + VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR | + (copy ? VK_PIPELINE_STAGE_2_TRANSFER_BIT : 0), + VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR | + (copy ? VK_PIPELINE_STAGE_2_TRANSFER_BIT : 0)); if (err < 0) { ff_vk_exec_discard(&ctx->s, exec); return err; @@ -523,29 +575,32 @@ int ff_vk_decode_frame(AVCodecContext *avctx, * picture then needs no host wait. */ ff_vk_exec_add_dep_refstruct(&ctx->s, exec, vp->out_views); - /* Output image - change layout, as it comes from a pool */ - img_bar[nb_img_bar] = (VkImageMemoryBarrier2) { - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, - .pNext = NULL, - .srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR, - .dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR, - .srcAccessMask = VK_ACCESS_2_NONE, - .dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR, - .oldLayout = vkf->layout[0], - .newLayout = (layered_dpb || vp->dpb_img) ? - VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR : - VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR, /* Spec, 07252 utter madness */ - .srcQueueFamilyIndex = vkf->queue_family[0], - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = vkf->img[0], - .subresourceRange = (VkImageSubresourceRange) { - .aspectMask = vp->view.aspect, - .layerCount = 1, - .levelCount = 1, - }, - }; - ff_vk_exec_update_frame(&ctx->s, exec, pic, - &img_bar[nb_img_bar], &nb_img_bar); + /* Output image - change layout, as it comes from a pool. + * Coincide copy-out decodes into the DPB layer, not this frame. */ + if (!copy) { + img_bar[nb_img_bar] = (VkImageMemoryBarrier2) { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .pNext = NULL, + .srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR, + .dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR, + .srcAccessMask = VK_ACCESS_2_NONE, + .dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR, + .oldLayout = vkf->layout[0], + .newLayout = (layered_dpb || vp->dpb_img) ? + VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR : + VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR, /* Spec, 07252 utter madness */ + .srcQueueFamilyIndex = vkf->queue_family[0], + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vkf->img[0], + .subresourceRange = (VkImageSubresourceRange) { + .aspectMask = vp->view.aspect, + .layerCount = 1, + .levelCount = 1, + }, + }; + ff_vk_exec_update_frame(&ctx->s, exec, pic, + &img_bar[nb_img_bar], &nb_img_bar); + } /* Current picture's DISTINCT-mode DPB image: transition on first use */ if (vp->dpb_img) { @@ -647,10 +702,68 @@ int ff_vk_decode_frame(AVCodecContext *avctx, }); /* Start, use parameters, decode and end decoding */ + if (copy) + vp->decode_info.dstPictureResource = vp->ref; vk->CmdBeginVideoCodingKHR(cmd_buf, &decode_start); vk->CmdDecodeVideoKHR(cmd_buf, &vp->decode_info); vk->CmdEndVideoCodingKHR(cmd_buf, &decode_end); + if (copy) { + FFVkVideoDPBImage *dpb = ctx->common.layered_img; + uint32_t layer = vp->ref.baseArrayLayer; + VkImageAspectFlags as = ctx->common.layered_aspect; + + nb_img_bar = 0; + img_bar[nb_img_bar++] = (VkImageMemoryBarrier2) { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, + .oldLayout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR, + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + .image = dpb->img, + .subresourceRange = { as, 0, 1, layer, 1 }, + }; + ff_vk_frame_barrier(&ctx->s, exec, pic, img_bar, &nb_img_bar, + VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR, + VK_PIPELINE_STAGE_2_TRANSFER_BIT, + VK_ACCESS_2_TRANSFER_WRITE_BIT, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + VK_QUEUE_FAMILY_IGNORED); + vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) { + .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, + .pImageMemoryBarriers = img_bar, + .imageMemoryBarrierCount = nb_img_bar, + }); + vk_decode_copy_layer(vk, cmd_buf, dpb->img, layer, pic, + vp->ref.codedExtent); + nb_img_bar = 0; + img_bar[nb_img_bar++] = (VkImageMemoryBarrier2) { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR, + .srcAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, + .dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_READ_BIT_KHR | + VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR, + .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + .newLayout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR, + .image = dpb->img, + .subresourceRange = { as, 0, 1, layer, 1 }, + }; + ff_vk_frame_barrier(&ctx->s, exec, pic, img_bar, &nb_img_bar, + VK_PIPELINE_STAGE_2_TRANSFER_BIT, + VK_PIPELINE_STAGE_2_TRANSFER_BIT, + VK_ACCESS_2_MEMORY_READ_BIT, + VK_IMAGE_LAYOUT_GENERAL, + VK_QUEUE_FAMILY_IGNORED); + vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) { + .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, + .pImageMemoryBarriers = img_bar, + .imageMemoryBarrierCount = nb_img_bar, + }); + } + /* End recording and submit for execution */ return ff_vk_exec_submit(&ctx->s, exec); } @@ -1015,15 +1128,19 @@ static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_ VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR && !(caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR)) { av_log(avctx, AV_LOG_VERBOSE, - "COINCIDE-only decode without SEPARATE_REFERENCE_IMAGES " - "(layered DPB capability); continuing\n"); + "COINCIDE-only decode without SEPARATE_REFERENCE_IMAGES; " + "using layered DPB and copy-out\n"); } - dec->dedicated_dpb = !!(dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR); - ctx->common.layered_dpb = !dec->dedicated_dpb ? 0 : - !(caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR); + { + const int no_sep = !(caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR); + dec->dedicated_dpb = !!(dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR) || + (!!(dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR) && no_sep); + ctx->common.layered_dpb = dec->dedicated_dpb && no_sep; + } - if (dec->dedicated_dpb) { + if (dec->dedicated_dpb && + (dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR)) { fmt_info.imageUsage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR; } else { fmt_info.imageUsage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR | @@ -1202,13 +1319,13 @@ int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx) VK_IMAGE_USAGE_SAMPLED_BIT; if (prof) { - FFVulkanDecodeShared *ctx; + FFVulkanDecodeShared *ctx = dec->shared_ctx; hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR; if (!dec->dedicated_dpb) hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR; - - ctx = dec->shared_ctx; + if (vk_decode_copy_out(ctx)) + hwfc->usage |= VK_IMAGE_USAGE_TRANSFER_DST_BIT; if ((ctx->s.extensions & FF_VK_EXT_VIDEO_ENCODE_QUEUE) && (ctx->s.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_1)) hwfc->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR; @@ -1341,6 +1458,7 @@ int ff_vk_decode_init(AVCodecContext *avctx) /* Out-of-place decoding: create the internal, queue-exclusive DPB pool */ if (dec->dedicated_dpb || avctx->codec_id == AV_CODEC_ID_AV1) { VkImageTiling tiling = VK_IMAGE_TILING_OPTIMAL; + VkImageUsageFlags dpb_usage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR; void *create_pnext = (void *)ff_vk_find_struct(ctx->s.hwfc->create_pnext, VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR); /* Reference (DPB) images use the same tiling and pNext chain as output. @@ -1354,8 +1472,12 @@ int ff_vk_decode_init(AVCodecContext *avctx) av_assert2(ff_vk_find_struct(drm_create_pnext, VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR)); } + if (vk_decode_copy_out(ctx)) + dpb_usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR | + VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + err = ff_vk_video_dpb_init(s, &ctx->common, s->hwfc->format[0], - VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR, + dpb_usage, tiling, create_pnext, s->frames->width, s->frames->height, ctx->common.layered_dpb ? -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
