Module: Mesa Branch: main Commit: 4b5e26e678210481a1fcfddc428280a01e6cd2b5 URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=4b5e26e678210481a1fcfddc428280a01e6cd2b5
Author: Mike Blumenkrantz <[email protected]> Date: Wed Jun 9 19:14:35 2021 -0400 zink: split gfx shader cache based on stages present this makes the pool of progs in a given hash smaller and also minimizes the hashing required for a given program Reviewed-by: Hoe Hao Cheng <[email protected]> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12532> --- src/gallium/drivers/zink/zink_compiler.c | 2 +- src/gallium/drivers/zink/zink_context.c | 16 ++------- src/gallium/drivers/zink/zink_context.h | 5 ++- src/gallium/drivers/zink/zink_draw.cpp | 61 ++++++++++++++++++++++++++++++-- 4 files changed, 66 insertions(+), 18 deletions(-) diff --git a/src/gallium/drivers/zink/zink_compiler.c b/src/gallium/drivers/zink/zink_compiler.c index b2026f31f6e..8e5d0e80cd9 100644 --- a/src/gallium/drivers/zink/zink_compiler.c +++ b/src/gallium/drivers/zink/zink_compiler.c @@ -1079,7 +1079,7 @@ zink_shader_free(struct zink_context *ctx, struct zink_shader *shader) enum pipe_shader_type pstage = pipe_shader_type_from_mesa(shader->nir->info.stage); assert(pstage < ZINK_SHADER_COUNT); if (shader->nir->info.stage != MESA_SHADER_TESS_CTRL || !shader->is_generated) - _mesa_hash_table_remove_key(&ctx->program_cache, prog->shaders); + _mesa_hash_table_remove_key(&ctx->program_cache[prog->stages_present >> 2], prog->shaders); prog->shaders[pstage] = NULL; if (shader->nir->info.stage == MESA_SHADER_TESS_EVAL && shader->generated) /* automatically destroy generated tcs shaders when tes is destroyed */ diff --git a/src/gallium/drivers/zink/zink_context.c b/src/gallium/drivers/zink/zink_context.c index 1b8859a694f..f5521fecbaf 100644 --- a/src/gallium/drivers/zink/zink_context.c +++ b/src/gallium/drivers/zink/zink_context.c @@ -125,7 +125,8 @@ zink_context_destroy(struct pipe_context *pctx) u_upload_destroy(pctx->stream_uploader); u_upload_destroy(pctx->const_uploader); slab_destroy_child(&ctx->transfer_pool); - _mesa_hash_table_clear(&ctx->program_cache, NULL); + for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_cache); i++) + _mesa_hash_table_clear(&ctx->program_cache[i], NULL); _mesa_hash_table_clear(&ctx->compute_program_cache, NULL); _mesa_hash_table_destroy(ctx->render_pass_cache, NULL); slab_destroy_child(&ctx->transfer_pool_unsync); @@ -2461,18 +2462,6 @@ zink_shader_stage(enum pipe_shader_type type) return stages[type]; } -static uint32_t -hash_gfx_program(const void *key) -{ - return _mesa_hash_data(key, sizeof(struct zink_shader *) * (ZINK_SHADER_COUNT)); -} - -static bool -equals_gfx_program(const void *a, const void *b) -{ - return memcmp(a, b, sizeof(struct zink_shader *) * (ZINK_SHADER_COUNT)) == 0; -} - static void zink_flush(struct pipe_context *pctx, struct pipe_fence_handle **pfence, @@ -3526,7 +3515,6 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags) if (!ctx->blitter) goto fail; - _mesa_hash_table_init(&ctx->program_cache, ctx, hash_gfx_program, equals_gfx_program); _mesa_hash_table_init(&ctx->compute_program_cache, ctx, _mesa_hash_pointer, _mesa_key_pointer_equal); ctx->render_pass_cache = _mesa_hash_table_create(NULL, hash_render_pass_state, diff --git a/src/gallium/drivers/zink/zink_context.h b/src/gallium/drivers/zink/zink_context.h index 74cf6db2de3..5aeb7d56dcb 100644 --- a/src/gallium/drivers/zink/zink_context.h +++ b/src/gallium/drivers/zink/zink_context.h @@ -203,7 +203,10 @@ struct zink_context { bool shader_reads_basevertex; struct zink_gfx_pipeline_state gfx_pipeline_state; enum pipe_prim_type gfx_prim_mode; - struct hash_table program_cache; + /* there are 5 gfx stages, but VS and FS are assumed to be always present, + * thus only 3 stages need to be considered, giving 2^3 = 8 program caches. + */ + struct hash_table program_cache[8]; struct zink_gfx_program *curr_program; struct zink_descriptor_data *dd; diff --git a/src/gallium/drivers/zink/zink_draw.cpp b/src/gallium/drivers/zink/zink_draw.cpp index dca3f39b7a5..e8c262908c2 100644 --- a/src/gallium/drivers/zink/zink_draw.cpp +++ b/src/gallium/drivers/zink/zink_draw.cpp @@ -203,13 +203,13 @@ update_gfx_program(struct zink_context *ctx) unsigned bits = u_bit_consecutive(PIPE_SHADER_VERTEX, 5); if (ctx->dirty_shader_stages & bits) { struct zink_gfx_program *prog = NULL; - struct hash_entry *entry = _mesa_hash_table_search(&ctx->program_cache, + struct hash_entry *entry = _mesa_hash_table_search(&ctx->program_cache[ctx->shader_stages >> 2], ctx->gfx_stages); if (entry) zink_update_gfx_program(ctx, (struct zink_gfx_program*)entry->data); else { prog = zink_create_gfx_program(ctx, ctx->gfx_stages); - entry = _mesa_hash_table_insert(&ctx->program_cache, prog->shaders, prog); + entry = _mesa_hash_table_insert(&ctx->program_cache[ctx->shader_stages >> 2], prog->shaders, prog); } prog = (struct zink_gfx_program*)(entry ? entry->data : NULL); if (prog && prog != ctx->curr_program) { @@ -874,6 +874,54 @@ zink_invalid_launch_grid(struct pipe_context *pctx, const struct pipe_grid_info unreachable("compute shader not bound"); } +template <unsigned STAGE_MASK> +static uint32_t +hash_gfx_program(const void *key) +{ + const void **shaders = (const void**)key; + uint32_t base_hash = _mesa_hash_data(key, sizeof(void*) * 2); + uint32_t gs_hash = _mesa_hash_data(key, sizeof(void*) * 3); + if (STAGE_MASK == 0) //VS+FS + return base_hash; + if (STAGE_MASK == 1) //VS+GS+FS + return gs_hash; + /*VS+TCS+FS isn't a thing */ + /*VS+TCS+GS+FS isn't a thing */ + if (STAGE_MASK == 4) //VS+TES+FS + return XXH32(&shaders[PIPE_SHADER_TESS_EVAL], sizeof(void*), base_hash); + if (STAGE_MASK == 5) //VS+TES+GS+FS + return XXH32(&shaders[PIPE_SHADER_TESS_EVAL], sizeof(void*), gs_hash); + if (STAGE_MASK == 6) //VS+TCS+TES+FS + return XXH32(&shaders[PIPE_SHADER_TESS_CTRL], sizeof(void*) * 2, base_hash); + + /* all stages */ + return _mesa_hash_data(key, sizeof(void*) * ZINK_SHADER_COUNT); +} + +template <unsigned STAGE_MASK> +static bool +equals_gfx_program(const void *a, const void *b) +{ + const void **sa = (const void**)a; + const void **sb = (const void**)b; + if (STAGE_MASK == 0) //VS+FS + return !memcmp(a, b, sizeof(void*) * 2); + if (STAGE_MASK == 1) //VS+GS+FS + return !memcmp(a, b, sizeof(void*) * 3); + /*VS+TCS+FS isn't a thing */ + /*VS+TCS+GS+FS isn't a thing */ + if (STAGE_MASK == 4) //VS+TES+FS + return sa[PIPE_SHADER_TESS_EVAL] == sb[PIPE_SHADER_TESS_EVAL] && !memcmp(a, b, sizeof(void*) * 2); + if (STAGE_MASK == 5) //VS+TES+GS+FS + return sa[PIPE_SHADER_TESS_EVAL] == sb[PIPE_SHADER_TESS_EVAL] && !memcmp(a, b, sizeof(void*) * 3); + if (STAGE_MASK == 6) //VS+TCS+TES+FS + return !memcmp(&sa[PIPE_SHADER_TESS_CTRL], &sb[PIPE_SHADER_TESS_CTRL], sizeof(void*) * 2) && + !memcmp(a, b, sizeof(void*) * 2); + + /* all stages */ + return !memcmp(a, b, sizeof(void*) * ZINK_SHADER_COUNT); +} + extern "C" void zink_init_draw_functions(struct zink_context *ctx, struct zink_screen *screen) @@ -888,6 +936,15 @@ zink_init_draw_functions(struct zink_context *ctx, struct zink_screen *screen) * initialization of callbacks in upper layers (such as u_threaded_context). */ ctx->base.draw_vbo = zink_invalid_draw_vbo; + + _mesa_hash_table_init(&ctx->program_cache[0], ctx, hash_gfx_program<0>, equals_gfx_program<0>); + _mesa_hash_table_init(&ctx->program_cache[1], ctx, hash_gfx_program<1>, equals_gfx_program<1>); + _mesa_hash_table_init(&ctx->program_cache[2], ctx, hash_gfx_program<2>, equals_gfx_program<2>); + _mesa_hash_table_init(&ctx->program_cache[3], ctx, hash_gfx_program<3>, equals_gfx_program<3>); + _mesa_hash_table_init(&ctx->program_cache[4], ctx, hash_gfx_program<4>, equals_gfx_program<4>); + _mesa_hash_table_init(&ctx->program_cache[5], ctx, hash_gfx_program<5>, equals_gfx_program<5>); + _mesa_hash_table_init(&ctx->program_cache[6], ctx, hash_gfx_program<6>, equals_gfx_program<6>); + _mesa_hash_table_init(&ctx->program_cache[7], ctx, hash_gfx_program<7>, equals_gfx_program<7>); } void
