The scratch buffer will be used for private memory and also register spilling. --- src/gallium/drivers/radeonsi/si_compute.c | 85 ++++++++++++++++++++++++++++++- src/gallium/drivers/radeonsi/si_shader.c | 5 ++ src/gallium/drivers/radeonsi/si_shader.h | 2 + 3 files changed, 90 insertions(+), 2 deletions(-)
diff --git a/src/gallium/drivers/radeonsi/si_compute.c b/src/gallium/drivers/radeonsi/si_compute.c index a7d61e7..d6cbbf4 100644 --- a/src/gallium/drivers/radeonsi/si_compute.c +++ b/src/gallium/drivers/radeonsi/si_compute.c @@ -122,6 +122,43 @@ static void si_set_global_binding( } } +/** + * This function computes the value for R_00B860_COMPUTE_TMPRING_SIZE.WAVES + * /p block_layout is the number of threads in each work group. + * /p grid layout is the number of work groups. + */ +static unsigned compute_num_waves_for_scratch( + const struct radeon_info *info; + const uint *block_layout, + const uint *grid_layout) +{ + unsigned num_sh = MAX2(info->max_sh_per_se, 1); + unsigned num_se = MAX2(info->max_se, 1); + unsigned num_blocks = 1; + unsigned threads_per_block = 1; + unsigned waves_per_block; + unsigned waves_per_sh; + unsigned waves; + unsigned scratch_waves; + unsigned i; + + for (i = 0; i < 3; i++) { + threads_per_block *= block_layout[i]; + num_blocks *= grid_layout[i]; + } + + waves_per_block = align(threads_per_block, 64) / 64; + waves = waves_per_block * num_blocks; + waves_per_sh = align(waves, num_sh * num_se) / (num_sh * num_se); + scratch_waves = waves_per_sh * num_sh * num_se; + + if (waves_per_block > waves_per_sh) { + scratch_waves = waves_per_block * num_sh * num_se; + } + + return scratch_waves; +} + static void si_launch_grid( struct pipe_context *ctx, const uint *block_layout, const uint *grid_layout, @@ -134,13 +171,16 @@ static void si_launch_grid( unsigned kernel_args_size; unsigned num_work_size_bytes = 36; uint32_t kernel_args_offset = 0; + uint32_t scratch_offset = 0; uint32_t *kernel_args; uint64_t kernel_args_va; + uint64_t scratch_buffer_va = 0; uint64_t shader_va; unsigned arg_user_sgpr_count = NUM_USER_SGPRS; unsigned i; struct si_pipe_shader *shader = &program->kernels[pc]; unsigned lds_blocks; + unsigned num_waves_for_scratch; pm4->compute_pkt = true; si_cmd_context_control(pm4); @@ -158,7 +198,9 @@ static void si_launch_grid( /* Upload the kernel arguments */ /* The extra num_work_size_bytes are for work group / work item size information */ - kernel_args_size = program->input_size + num_work_size_bytes; + kernel_args_size = program->input_size + num_work_size_bytes + 8 /* For scratch va */; + scratch_offset = program->input_size + num_work_size_bytes; + kernel_args = MALLOC(kernel_args_size); for (i = 0; i < 3; i++) { kernel_args[i] = grid_layout[i]; @@ -166,8 +208,34 @@ static void si_launch_grid( kernel_args[i + 6] = block_layout[i]; } + num_waves_for_scratch = compute_num_waves_for_scratch( + &stcx->screen.info, block_layout, grid_layout); + memcpy(kernel_args + (num_work_size_bytes / 4), input, program->input_size); + if (shader->scratch_bytes_per_wave > 0) { + float *ptr; + + COMPUTE_DBG(sctx->screen, "Waves: %u; Scratch per wave: %u bytes; " + "Total Scratch: %u bytes\n", num_waves_for_scratch, + shader->scratch_bytes_per_wave, info.width0); + if (!shader->scratch_bo) { + shader->scratch_bo = (struct r600_resource*) + si_resource_create_custom(sctx->b.b.screen, + PIPE_USAGE_DEFAULT, info.width0); + } + ptr = sctx->b.ws->buffer_map(shader->scratch_bo->cs_buf, sctx->b.rings.gfx.cs, + PIPE_TRANSFER_READ); + scratch_buffer_va = r600_resource_va(ctx->screen, + (struct pipe_resource*)shader->scratch_bo); + si_pm4_add_bo(pm4, shader->scratch_bo, + RADEON_USAGE_READWRITE, + RADEON_PRIO_SHADER_RESOURCE_RW); + + } + memcpy(kernel_args + (scratch_offset / 4), &scratch_buffer_va, + sizeof(scratch_buffer_va)); + for (i = 0; i < (kernel_args_size / 4); i++) { COMPUTE_DBG(sctx->screen, "input %u : %u\n", i, kernel_args[i]); @@ -183,6 +251,10 @@ static void si_launch_grid( si_pm4_set_reg(pm4, R_00B900_COMPUTE_USER_DATA_0, kernel_args_va); si_pm4_set_reg(pm4, R_00B900_COMPUTE_USER_DATA_0 + 4, S_008F04_BASE_ADDRESS_HI (kernel_args_va >> 32) | S_008F04_STRIDE(0)); + si_pm4_set_reg(pm4, R_00B900_COMPUTE_USER_DATA_0 + 8, scratch_buffer_va); + si_pm4_set_reg(pm4, R_00B900_COMPUTE_USER_DATA_0 + 12, + S_008F04_BASE_ADDRESS_HI(scratch_buffer_va >> 32) + | S_008F04_STRIDE(shader->scratch_bytes_per_wave / 64)); si_pm4_set_reg(pm4, R_00B810_COMPUTE_START_X, 0); si_pm4_set_reg(pm4, R_00B814_COMPUTE_START_Y, 0); @@ -252,7 +324,7 @@ static void si_launch_grid( assert(lds_blocks <= 0xFF); si_pm4_set_reg(pm4, R_00B84C_COMPUTE_PGM_RSRC2, - S_00B84C_SCRATCH_EN(0) + S_00B84C_SCRATCH_EN(shader->scratch_bytes_per_wave > 0) | S_00B84C_USER_SGPR(arg_user_sgpr_count) | S_00B84C_TGID_X_EN(1) | S_00B84C_TGID_Y_EN(1) @@ -274,6 +346,15 @@ static void si_launch_grid( | S_00B85C_SH1_CU_EN(0xffff /* Default value */)) ; + si_pm4_set_reg(pm4, R_00B860_COMPUTE_TMPRING_SIZE, + /* The maximum value for WAVES is 32 * num CU. + * If you program this value incorrectly, the GPU will hang if + * COMPUTE_PGM_RSRC2.SCRATCH_EN is enabled. + */ + S_00B860_WAVES(num_waves_for_scratch) + | S_00B860_WAVESIZE(shader->scratch_bytes_per_wave >> 10)) + ; + si_pm4_cmd_begin(pm4, PKT3_DISPATCH_DIRECT); si_pm4_cmd_add(pm4, grid_layout[0]); /* Thread groups DIM_X */ si_pm4_cmd_add(pm4, grid_layout[1]); /* Thread groups DIM_Y */ diff --git a/src/gallium/drivers/radeonsi/si_shader.c b/src/gallium/drivers/radeonsi/si_shader.c index 641e563..ae0da2a 100644 --- a/src/gallium/drivers/radeonsi/si_shader.c +++ b/src/gallium/drivers/radeonsi/si_shader.c @@ -2544,6 +2544,11 @@ int si_compile_llvm(struct si_context *sctx, struct si_pipe_shader *shader, case R_0286CC_SPI_PS_INPUT_ENA: shader->spi_ps_input_ena = value; break; + case R_00B860_COMPUTE_TMPRING_SIZE: + /* WAVESIZE is in units of 256 dwords. */ + shader->scratch_bytes_per_wave = + G_00B860_WAVESIZE(value) * 256 * 4 * 1; + break; default: fprintf(stderr, "Warning: Compiler emitted unknown " "config register: 0x%x\n", reg); diff --git a/src/gallium/drivers/radeonsi/si_shader.h b/src/gallium/drivers/radeonsi/si_shader.h index 81997c0..57460d3 100644 --- a/src/gallium/drivers/radeonsi/si_shader.h +++ b/src/gallium/drivers/radeonsi/si_shader.h @@ -176,10 +176,12 @@ struct si_pipe_shader { struct si_shader shader; struct si_pm4_state *pm4; struct r600_resource *bo; + struct r600_resource *scratch_bo; unsigned num_sgprs; unsigned num_vgprs; unsigned lds_size; unsigned spi_ps_input_ena; + unsigned scratch_bytes_per_wave; unsigned spi_shader_col_format; unsigned spi_shader_z_format; unsigned db_shader_control; -- 1.8.1.5 _______________________________________________ mesa-dev mailing list mesa-dev@lists.freedesktop.org http://lists.freedesktop.org/mailman/listinfo/mesa-dev