Module: Mesa Branch: main Commit: 61496915c2182cb623f2cd175878a4c98f176d55 URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=61496915c2182cb623f2cd175878a4c98f176d55
Author: Lionel Landwerlin <[email protected]> Date: Thu Jun 15 13:44:44 2023 +0300 anv: tracking query buffer writes & query clears separately Clears should be flushed only on : - vkCmdBeginQuery* - vkCmdWriteTimestamp* - vkCmdWriteAccelerationStructuresPropertiesKHR - vkCmdCopyQueryPoolResults Buffer writes should be flush only on : - vkCmdCopyQueryPoolResults Signed-off-by: Lionel Landwerlin <[email protected]> Reviewed-by: Rohan Garg <[email protected]> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/23675> --- src/intel/vulkan/anv_blorp.c | 2 +- src/intel/vulkan/anv_cmd_buffer.c | 28 ++++++++++++++++++---------- src/intel/vulkan/anv_private.h | 26 +++++++++++++++++++------- src/intel/vulkan/genX_cmd_buffer.c | 16 ++++++++-------- src/intel/vulkan/genX_query.c | 24 +++++++++++++++--------- 5 files changed, 61 insertions(+), 35 deletions(-) diff --git a/src/intel/vulkan/anv_blorp.c b/src/intel/vulkan/anv_blorp.c index d51c180680f..e079617375e 100644 --- a/src/intel/vulkan/anv_blorp.c +++ b/src/intel/vulkan/anv_blorp.c @@ -541,7 +541,7 @@ anv_add_buffer_write_pending_bits(struct anv_cmd_buffer *cmd_buffer, { const struct intel_device_info *devinfo = cmd_buffer->device->info; - cmd_buffer->state.pending_query_bits |= + cmd_buffer->state.queries.buffer_write_bits |= (cmd_buffer->queue_family->queueFlags & VK_QUEUE_GRAPHICS_BIT) == 0 ? ANV_QUERY_COMPUTE_WRITES_PENDING_BITS : ANV_QUERY_RENDER_TARGET_WRITES_PENDING_BITS(devinfo); diff --git a/src/intel/vulkan/anv_cmd_buffer.c b/src/intel/vulkan/anv_cmd_buffer.c index 0daae05984b..909f63d737c 100644 --- a/src/intel/vulkan/anv_cmd_buffer.c +++ b/src/intel/vulkan/anv_cmd_buffer.c @@ -298,27 +298,35 @@ anv_cmd_emit_conditional_render_predicate(struct anv_cmd_buffer *cmd_buffer) anv_genX(devinfo, cmd_emit_conditional_render_predicate)(cmd_buffer); } -void -anv_cmd_buffer_update_pending_query_bits(struct anv_cmd_buffer *cmd_buffer, - enum anv_pipe_bits flushed_bits) +static void +clear_pending_query_bits(enum anv_query_bits *query_bits, + enum anv_pipe_bits flushed_bits) { if (flushed_bits & ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT) - cmd_buffer->state.pending_query_bits &= ~ANV_QUERY_WRITES_RT_FLUSH; + *query_bits &= ~ANV_QUERY_WRITES_RT_FLUSH; if (flushed_bits & ANV_PIPE_TILE_CACHE_FLUSH_BIT) - cmd_buffer->state.pending_query_bits &= ~ANV_QUERY_WRITES_TILE_FLUSH; + *query_bits &= ~ANV_QUERY_WRITES_TILE_FLUSH; if ((flushed_bits & ANV_PIPE_DATA_CACHE_FLUSH_BIT) && (flushed_bits & ANV_PIPE_HDC_PIPELINE_FLUSH_BIT) && (flushed_bits & ANV_PIPE_UNTYPED_DATAPORT_CACHE_FLUSH_BIT)) - cmd_buffer->state.pending_query_bits &= ~ANV_QUERY_WRITES_TILE_FLUSH; + *query_bits &= ~ANV_QUERY_WRITES_TILE_FLUSH; /* Once RT/TILE have been flushed, we can consider the CS_STALL flush */ - if ((cmd_buffer->state.pending_query_bits & (ANV_QUERY_WRITES_TILE_FLUSH | - ANV_QUERY_WRITES_RT_FLUSH | - ANV_QUERY_WRITES_DATA_FLUSH)) == 0 && + if ((*query_bits & (ANV_QUERY_WRITES_TILE_FLUSH | + ANV_QUERY_WRITES_RT_FLUSH | + ANV_QUERY_WRITES_DATA_FLUSH)) == 0 && (flushed_bits & (ANV_PIPE_END_OF_PIPE_SYNC_BIT | ANV_PIPE_CS_STALL_BIT))) - cmd_buffer->state.pending_query_bits &= ~ANV_QUERY_WRITES_CS_STALL; + *query_bits &= ~ANV_QUERY_WRITES_CS_STALL; +} + +void +anv_cmd_buffer_update_pending_query_bits(struct anv_cmd_buffer *cmd_buffer, + enum anv_pipe_bits flushed_bits) +{ + clear_pending_query_bits(&cmd_buffer->state.queries.clear_bits, flushed_bits); + clear_pending_query_bits(&cmd_buffer->state.queries.buffer_write_bits, flushed_bits); } static bool diff --git a/src/intel/vulkan/anv_private.h b/src/intel/vulkan/anv_private.h index 1631184d2fd..c6be276716b 100644 --- a/src/intel/vulkan/anv_private.h +++ b/src/intel/vulkan/anv_private.h @@ -2850,13 +2850,25 @@ struct anv_cmd_state { enum anv_pipe_bits pending_pipe_bits; - /** - * Tracks operations susceptible to interfere with queries, either blorp - * clears of the query buffer or the destination buffer of - * vkCmdCopyQueryResults, we need those operations to have completed before - * we do the work of vkCmdCopyQueryResults. - */ - enum anv_query_bits pending_query_bits; + struct { + /** + * Tracks operations susceptible to interfere with queries in the + * destination buffer of vkCmdCopyQueryResults, we need those operations to + * have completed before we do the work of vkCmdCopyQueryResults. + */ + enum anv_query_bits buffer_write_bits; + + /** + * Tracks clear operations of query buffers that can interact with + * vkCmdQueryBegin*, vkCmdWriteTimestamp*, + * vkCmdWriteAccelerationStructuresPropertiesKHR, etc... + * + * We need the clearing of the buffer completed before with write data with + * the command streamer or a shader. + */ + enum anv_query_bits clear_bits; + } queries; + VkShaderStageFlags descriptors_dirty; VkShaderStageFlags push_descriptors_dirty; VkShaderStageFlags push_constants_dirty; diff --git a/src/intel/vulkan/genX_cmd_buffer.c b/src/intel/vulkan/genX_cmd_buffer.c index 22cb43392e2..23441ad4f31 100644 --- a/src/intel/vulkan/genX_cmd_buffer.c +++ b/src/intel/vulkan/genX_cmd_buffer.c @@ -3839,9 +3839,9 @@ genX(EndCommandBuffer)( /* Flush query clears using blorp so that secondary query writes do not * race with the clear. */ - if (cmd_buffer->state.pending_query_bits) { + if (cmd_buffer->state.queries.clear_bits) { anv_add_pending_pipe_bits(cmd_buffer, - ANV_PIPE_QUERY_BITS(cmd_buffer->state.pending_query_bits), + ANV_PIPE_QUERY_BITS(cmd_buffer->state.queries.clear_bits), "query clear flush prior command buffer end"); } @@ -3923,9 +3923,9 @@ genX(CmdExecuteCommands)( /* Flush query clears using blorp so that secondary query writes do not * race with the clear. */ - if (primary->state.pending_query_bits) { + if (primary->state.queries.clear_bits) { anv_add_pending_pipe_bits(primary, - ANV_PIPE_QUERY_BITS(primary->state.pending_query_bits), + ANV_PIPE_QUERY_BITS(primary->state.queries.clear_bits), "query clear flush prior to secondary buffer"); } @@ -4099,7 +4099,7 @@ cmd_buffer_barrier(struct anv_cmd_buffer *cmd_buffer, if (stage_is_shader(dep_info->pMemoryBarriers[i].srcStageMask) && mask_is_shader_write(dep_info->pMemoryBarriers[i].srcAccessMask) && stage_is_transfer(dep_info->pMemoryBarriers[i].dstStageMask)) { - cmd_buffer->state.pending_query_bits |= + cmd_buffer->state.queries.buffer_write_bits |= ANV_QUERY_COMPUTE_WRITES_PENDING_BITS; } } @@ -4114,7 +4114,7 @@ cmd_buffer_barrier(struct anv_cmd_buffer *cmd_buffer, if (stage_is_shader(dep_info->pBufferMemoryBarriers[i].srcStageMask) && mask_is_shader_write(dep_info->pBufferMemoryBarriers[i].srcAccessMask) && stage_is_transfer(dep_info->pBufferMemoryBarriers[i].dstStageMask)) { - cmd_buffer->state.pending_query_bits |= + cmd_buffer->state.queries.buffer_write_bits |= ANV_QUERY_COMPUTE_WRITES_PENDING_BITS; } } @@ -6716,9 +6716,9 @@ genX(flush_pipeline_select)(struct anv_cmd_buffer *cmd_buffer, * copy/write. So we need to flush it before going to GPGPU mode. */ if (cmd_buffer->state.current_pipeline == _3D && - cmd_buffer->state.pending_query_bits) { + cmd_buffer->state.queries.clear_bits) { anv_add_pending_pipe_bits(cmd_buffer, - ANV_PIPE_QUERY_BITS(cmd_buffer->state.pending_query_bits), + ANV_PIPE_QUERY_BITS(cmd_buffer->state.queries.clear_bits), "query clear flush prior to GPGPU"); } #endif diff --git a/src/intel/vulkan/genX_query.c b/src/intel/vulkan/genX_query.c index 396fba29c4f..9d13b5babab 100644 --- a/src/intel/vulkan/genX_query.c +++ b/src/intel/vulkan/genX_query.c @@ -790,7 +790,7 @@ void genX(CmdResetQueryPool)( queryCount * pool->stride, 0); - cmd_buffer->state.pending_query_bits = + cmd_buffer->state.queries.clear_bits = (cmd_buffer->queue_family->queueFlags & VK_QUEUE_GRAPHICS_BIT) == 0 ? ANV_QUERY_COMPUTE_WRITES_PENDING_BITS : ANV_QUERY_RENDER_TARGET_WRITES_PENDING_BITS(cmd_buffer->device->info); @@ -987,12 +987,12 @@ emit_query_clear_flush(struct anv_cmd_buffer *cmd_buffer, struct anv_query_pool *pool, const char *reason) { - if (cmd_buffer->state.pending_query_bits == 0) + if (cmd_buffer->state.queries.clear_bits == 0) return; anv_add_pending_pipe_bits(cmd_buffer, ANV_PIPE_QUERY_BITS( - cmd_buffer->state.pending_query_bits), + cmd_buffer->state.queries.clear_bits), reason); genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer); } @@ -1507,22 +1507,26 @@ copy_query_results_with_cs(struct anv_cmd_buffer *cmd_buffer, * to ensure proper ordering of the commands from the 3d pipe and the * command streamer. */ - if (cmd_buffer->state.pending_query_bits & + if ((cmd_buffer->state.queries.buffer_write_bits | + cmd_buffer->state.queries.clear_bits) & ANV_QUERY_WRITES_RT_FLUSH) needed_flushes |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT; - if (cmd_buffer->state.pending_query_bits & + if ((cmd_buffer->state.queries.buffer_write_bits | + cmd_buffer->state.queries.clear_bits) & ANV_QUERY_WRITES_TILE_FLUSH) needed_flushes |= ANV_PIPE_TILE_CACHE_FLUSH_BIT; - if (cmd_buffer->state.pending_query_bits & + if ((cmd_buffer->state.queries.buffer_write_bits | + cmd_buffer->state.queries.clear_bits) & ANV_QUERY_WRITES_DATA_FLUSH) { needed_flushes |= (ANV_PIPE_DATA_CACHE_FLUSH_BIT | ANV_PIPE_HDC_PIPELINE_FLUSH_BIT | ANV_PIPE_UNTYPED_DATAPORT_CACHE_FLUSH_BIT); } - if (cmd_buffer->state.pending_query_bits & + if ((cmd_buffer->state.queries.buffer_write_bits | + cmd_buffer->state.queries.clear_bits) & ANV_QUERY_WRITES_CS_STALL) needed_flushes |= ANV_PIPE_CS_STALL_BIT; @@ -1661,10 +1665,12 @@ copy_query_results_with_shader(struct anv_cmd_buffer *cmd_buffer, if (cmd_buffer->state.current_pipeline == UINT32_MAX) genX(flush_pipeline_select_3d)(cmd_buffer); - if (cmd_buffer->state.pending_query_bits & ANV_QUERY_WRITES_RT_FLUSH) + if ((cmd_buffer->state.queries.buffer_write_bits | + cmd_buffer->state.queries.clear_bits) & ANV_QUERY_WRITES_RT_FLUSH) needed_flushes |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT; - if (cmd_buffer->state.pending_query_bits & ANV_QUERY_WRITES_DATA_FLUSH) { + if ((cmd_buffer->state.queries.buffer_write_bits | + cmd_buffer->state.queries.clear_bits) & ANV_QUERY_WRITES_DATA_FLUSH) { needed_flushes |= (ANV_PIPE_HDC_PIPELINE_FLUSH_BIT | ANV_PIPE_UNTYPED_DATAPORT_CACHE_FLUSH_BIT); }
