From: Marek Olšák <marek.ol...@amd.com>
---
src/amd/common/ac_llvm_build.c | 68 ++++++++++++++
src/amd/common/ac_llvm_build.h | 29 ++++++
src/gallium/drivers/radeonsi/si_shader.c | 154 +++++++++++++------------------
3 files changed, 159 insertions(+), 92 deletions(-)
diff --git a/src/amd/common/ac_llvm_build.c b/src/amd/common/ac_llvm_build.c
index bd1b63d..3a1ef93 100644
--- a/src/amd/common/ac_llvm_build.c
+++ b/src/amd/common/ac_llvm_build.c
@@ -925,10 +925,78 @@ void ac_emit_export(struct ac_llvm_context *ctx, struct
ac_export_args *a)
args[0] = LLVMConstInt(ctx->i32, a->enabled_channels, 0);
args[1] = LLVMConstInt(ctx->i32, a->valid_mask, 0);
args[2] = LLVMConstInt(ctx->i32, a->done, 0);
args[3] = LLVMConstInt(ctx->i32, a->target, 0);
args[4] = LLVMConstInt(ctx->i32, a->compr, 0);
memcpy(args + 5, a->out, sizeof(a->out[0]) * 4);
ac_emit_llvm_intrinsic(ctx, "llvm.SI.export", ctx->voidt, args, 9,
AC_FUNC_ATTR_LEGACY);
}
+
+LLVMValueRef ac_emit_image_opcode(struct ac_llvm_context *ctx,
+ struct ac_image_args *a)
+{
+ LLVMTypeRef dst_type;
+ LLVMValueRef args[11];
+ unsigned num_args = 0;
+ const char *name;
+ char intr_name[128], type[64];
+
+ args[num_args++] = a->addr;
+ args[num_args++] = a->resource;
+
+ if (a->opcode == ac_image_load ||
+ a->opcode == ac_image_load_mip ||
+ a->opcode == ac_image_get_resinfo) {
+ dst_type = ctx->v4i32;
+ } else {
+ dst_type = ctx->v4f32;
+ args[num_args++] = a->sampler;
+ }
+
+ args[num_args++] = LLVMConstInt(ctx->i32, a->dmask, 0);
+ args[num_args++] = LLVMConstInt(ctx->i32, a->unorm, 0);
+ args[num_args++] = LLVMConstInt(ctx->i32, 0, 0); /* r128 */
+ args[num_args++] = LLVMConstInt(ctx->i32, a->da, 0);
+ args[num_args++] = LLVMConstInt(ctx->i32, 0, 0); /* glc */
+ args[num_args++] = LLVMConstInt(ctx->i32, 0, 0); /* slc */
+ args[num_args++] = LLVMConstInt(ctx->i32, 0, 0); /* tfe */
+ args[num_args++] = LLVMConstInt(ctx->i32, 0, 0); /* lwe */
+
+ switch (a->opcode) {
+ case ac_image_sample:
+ name = "llvm.SI.image.sample";
+ break;
+ case ac_image_gather4:
+ name = "llvm.SI.gather4";
+ break;
+ case ac_image_load:
+ name = "llvm.SI.image.load";
+ break;
+ case ac_image_load_mip:
+ name = "llvm.SI.image.load.mip";
+ break;
+ case ac_image_get_lod:
+ name = "llvm.SI.getlod";
+ break;
+ case ac_image_get_resinfo:
+ name = "llvm.SI.getresinfo";
+ break;
+ }
+
+ ac_build_type_name_for_intr(LLVMTypeOf(a->addr), type, sizeof(type));
+ snprintf(intr_name, sizeof(intr_name), "%s%s%s%s.%s",
+ name,
+ a->compare ? ".c" : "",
+ a->bias ? ".b" :
+ a->lod ? ".l" :
+ a->deriv ? ".d" :
+ a->level_zero ? ".lz" : "",
+ a->offset ? ".o" : "",
+ type);
+
+ return ac_emit_llvm_intrinsic(ctx, intr_name,
+ dst_type, args, num_args,
+ AC_FUNC_ATTR_READNONE |
+ AC_FUNC_ATTR_LEGACY);
+}
diff --git a/src/amd/common/ac_llvm_build.h b/src/amd/common/ac_llvm_build.h
index 27f2097..f57acc2 100644
--- a/src/amd/common/ac_llvm_build.h
+++ b/src/amd/common/ac_llvm_build.h
@@ -198,15 +198,44 @@ struct ac_export_args {
LLVMValueRef out[4];
unsigned target;
unsigned enabled_channels;
bool compr;
bool done;
bool valid_mask;
};
void ac_emit_export(struct ac_llvm_context *ctx, struct ac_export_args *a);
+enum ac_image_opcode {
+ ac_image_sample,
+ ac_image_gather4,
+ ac_image_load,
+ ac_image_load_mip,
+ ac_image_get_lod,
+ ac_image_get_resinfo,
+};
+
+struct ac_image_args {
+ enum ac_image_opcode opcode;
+ bool level_zero;
+ bool bias;
+ bool lod;
+ bool deriv;
+ bool compare;
+ bool offset;
+
+ LLVMValueRef resource;
+ LLVMValueRef sampler;
+ LLVMValueRef addr;
+ unsigned dmask;
+ bool unorm;
+ bool da;
+};
+
+LLVMValueRef ac_emit_image_opcode(struct ac_llvm_context *ctx,
+ struct ac_image_args *a);
+
#ifdef __cplusplus
}
#endif
#endif
diff --git a/src/gallium/drivers/radeonsi/si_shader.c
b/src/gallium/drivers/radeonsi/si_shader.c
index 8c07b4f..082e071 100644
--- a/src/gallium/drivers/radeonsi/si_shader.c
+++ b/src/gallium/drivers/radeonsi/si_shader.c
@@ -4054,63 +4054,47 @@ static void resq_emit(
z = LLVMBuildSDiv(builder, z, imm6, "");
out = LLVMBuildInsertElement(builder, out, z, imm2, "");
}
}
emit_data->output[emit_data->chan] = out;
}
static void set_tex_fetch_args(struct si_shader_context *ctx,
struct lp_build_emit_data *emit_data,
- unsigned opcode, unsigned target,
+ unsigned target,
LLVMValueRef res_ptr, LLVMValueRef samp_ptr,
LLVMValueRef *param, unsigned count,
unsigned dmask)
{
struct gallivm_state *gallivm = &ctx->gallivm;
- unsigned num_args;
- unsigned is_rect = target == TGSI_TEXTURE_RECT ||
- target == TGSI_TEXTURE_SHADOWRECT;
+ struct ac_image_args args = {};
/* Pad to power of two vector */
while (count < util_next_power_of_two(count))
param[count++] = LLVMGetUndef(ctx->i32);
- /* Texture coordinates. */
if (count > 1)
- emit_data->args[0] = lp_build_gather_values(gallivm, param,
count);
+ args.addr = lp_build_gather_values(gallivm, param, count);
else
- emit_data->args[0] = param[0];
+ args.addr = param[0];
- /* Resource. */
- emit_data->args[1] = res_ptr;
- num_args = 2;
+ args.resource = res_ptr;
+ args.sampler = samp_ptr;
+ args.dmask = dmask;
+ args.unorm = target == TGSI_TEXTURE_RECT ||
+ target == TGSI_TEXTURE_SHADOWRECT;
+ args.da = tgsi_is_array_sampler(target);
- if (opcode == TGSI_OPCODE_TXF || opcode == TGSI_OPCODE_TXQ)
- emit_data->dst_type = ctx->v4i32;
- else {
- emit_data->dst_type = ctx->v4f32;
-
- emit_data->args[num_args++] = samp_ptr;
- }
-
- emit_data->args[num_args++] = lp_build_const_int32(gallivm, dmask);
- emit_data->args[num_args++] = lp_build_const_int32(gallivm, is_rect);
/* unorm */
- emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* r128
*/
- emit_data->args[num_args++] = lp_build_const_int32(gallivm,
- tgsi_is_array_sampler(target)); /* da */
- emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* glc
*/
- emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* slc
*/
- emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* tfe
*/
- emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* lwe
*/
-
- emit_data->arg_count = num_args;
+ /* Ugly, but we seem to have no other choice right now. */
+ STATIC_ASSERT(sizeof(args) <= sizeof(emit_data->args));
+ memcpy(emit_data->args, &args, sizeof(args));
}
static const struct lp_build_tgsi_action tex_action;
enum desc_type {
DESC_IMAGE,
DESC_BUFFER,
DESC_FMASK,
DESC_SAMPLER,
};
@@ -4254,41 +4238,43 @@ static void txq_fetch_args(
if (target == TGSI_TEXTURE_BUFFER) {
/* Read the size from the buffer descriptor directly. */
emit_data->args[0] = get_buffer_size(bld_base, res_ptr);
return;
}
/* Textures - set the mip level. */
address = lp_build_emit_fetch(bld_base, inst, 0, TGSI_CHAN_X);
- set_tex_fetch_args(ctx, emit_data, TGSI_OPCODE_TXQ, target, res_ptr,
+ set_tex_fetch_args(ctx, emit_data, target, res_ptr,
NULL, &address, 1, 0xf);
}
static void txq_emit(const struct lp_build_tgsi_action *action,
struct lp_build_tgsi_context *bld_base,
struct lp_build_emit_data *emit_data)
{
- struct lp_build_context *base = &bld_base->base;
+ struct si_shader_context *ctx = si_shader_context(bld_base);
+ struct ac_image_args args;
unsigned target = emit_data->inst->Texture.Texture;
if (target == TGSI_TEXTURE_BUFFER) {
/* Just return the buffer size. */
emit_data->output[emit_data->chan] = emit_data->args[0];
return;
}
- emit_data->output[emit_data->chan] = lp_build_intrinsic(
- base->gallivm->builder, "llvm.SI.getresinfo.i32",
- emit_data->dst_type, emit_data->args, emit_data->arg_count,
- LP_FUNC_ATTR_READNONE | LP_FUNC_ATTR_LEGACY);
+ memcpy(&args, emit_data->args, sizeof(args)); /* ugly */
+
+ args.opcode = ac_image_get_resinfo;
+ emit_data->output[emit_data->chan] =
+ ac_emit_image_opcode(&ctx->ac, &args);
/* Divide the number of layers by 6 to get the number of cubes. */
if (target == TGSI_TEXTURE_CUBE_ARRAY ||
target == TGSI_TEXTURE_SHADOWCUBE_ARRAY) {
LLVMBuilderRef builder = bld_base->base.gallivm->builder;
LLVMValueRef two = lp_build_const_int32(bld_base->base.gallivm,
2);
LLVMValueRef six = lp_build_const_int32(bld_base->base.gallivm,
6);
LLVMValueRef v4 = emit_data->output[emit_data->chan];
LLVMValueRef z = LLVMBuildExtractElement(builder, v4, two, "");
@@ -4504,21 +4490,21 @@ static void tex_fetch_args(
unsigned txf_count = target == TGSI_TEXTURE_2D_MSAA ? 2 : 3;
struct tgsi_full_instruction inst = {};
memcpy(txf_address, address, sizeof(txf_address));
/* Read FMASK using TXF. */
inst.Instruction.Opcode = TGSI_OPCODE_TXF;
inst.Texture.Texture = target;
txf_emit_data.inst = &inst;
txf_emit_data.chan = 0;
- set_tex_fetch_args(ctx, &txf_emit_data, TGSI_OPCODE_TXF,
+ set_tex_fetch_args(ctx, &txf_emit_data,
target, fmask_ptr, NULL,
txf_address, txf_count, 0xf);
build_tex_intrinsic(&tex_action, bld_base, &txf_emit_data);
/* Initialize some constants. */
LLVMValueRef four = LLVMConstInt(ctx->i32, 4, 0);
LLVMValueRef F = LLVMConstInt(ctx->i32, 0xF, 0);
/* Apply the formula. */
LLVMValueRef fmask =
@@ -4613,57 +4599,60 @@ static void tex_fetch_args(
assert(src1.File == TGSI_FILE_IMMEDIATE);
comp_imm = ctx->imms[src1.Index * TGSI_NUM_CHANNELS +
src1.SwizzleX];
gather_comp = LLVMConstIntGetZExtValue(comp_imm);
gather_comp = CLAMP(gather_comp, 0, 3);
}
dmask = 1 << gather_comp;
}
- set_tex_fetch_args(ctx, emit_data, opcode, target, res_ptr,
+ set_tex_fetch_args(ctx, emit_data, target, res_ptr,
samp_ptr, address, count, dmask);
}
/* Gather4 should follow the same rules as bilinear filtering, but the hardware
* incorrectly forces nearest filtering if the texture format is integer.
* The only effect it has on Gather4, which always returns 4 texels for
* bilinear filtering, is that the final coordinates are off by 0.5 of
* the texel size.
*
* The workaround is to subtract 0.5 from the unnormalized coordinates,
* or (0.5 / size) from the normalized coordinates.
*/
static void si_lower_gather4_integer(struct si_shader_context *ctx,
- struct lp_build_emit_data *emit_data,
- const char *intr_name,
- unsigned coord_vgpr_index)
+ struct ac_image_args *args,
+ unsigned target)
{
LLVMBuilderRef builder = ctx->gallivm.builder;
- LLVMValueRef coord = emit_data->args[0];
+ LLVMValueRef coord = args->addr;
LLVMValueRef half_texel[2];
+ /* Texture coordinates start after:
+ * {offset, bias, z-compare, derivatives}
+ * Only the offset and z-compare can occur here.
+ */
+ unsigned coord_vgpr_index = (int)args->offset + (int)args->compare;
int c;
- if (emit_data->inst->Texture.Texture == TGSI_TEXTURE_RECT ||
- emit_data->inst->Texture.Texture == TGSI_TEXTURE_SHADOWRECT) {
+ if (target == TGSI_TEXTURE_RECT ||
+ target == TGSI_TEXTURE_SHADOWRECT) {
half_texel[0] = half_texel[1] = LLVMConstReal(ctx->f32, -0.5);
} else {
struct tgsi_full_instruction txq_inst = {};
struct lp_build_emit_data txq_emit_data = {};
/* Query the texture size. */
- txq_inst.Texture.Texture = emit_data->inst->Texture.Texture;
+ txq_inst.Texture.Texture = target;
txq_emit_data.inst = &txq_inst;
txq_emit_data.dst_type = ctx->v4i32;
- set_tex_fetch_args(ctx, &txq_emit_data, TGSI_OPCODE_TXQ,
- txq_inst.Texture.Texture,
- emit_data->args[1], NULL,
+ set_tex_fetch_args(ctx, &txq_emit_data, target,
+ args->resource, NULL,
&ctx->bld_base.uint_bld.zero,
1, 0xf);
txq_emit(NULL, &ctx->bld_base, &txq_emit_data);
/* Compute -0.5 / size. */
for (c = 0; c < 2; c++) {
half_texel[c] =
LLVMBuildExtractElement(builder,
txq_emit_data.output[0],
LLVMConstInt(ctx->i32,
c, 0), "");
half_texel[c] = LLVMBuildUIToFP(builder, half_texel[c],
ctx->f32, "");
@@ -4679,126 +4668,107 @@ static void si_lower_gather4_integer(struct
si_shader_context *ctx,
LLVMValueRef tmp;
LLVMValueRef index = LLVMConstInt(ctx->i32, coord_vgpr_index +
c, 0);
tmp = LLVMBuildExtractElement(builder, coord, index, "");
tmp = LLVMBuildBitCast(builder, tmp, ctx->f32, "");
tmp = LLVMBuildFAdd(builder, tmp, half_texel[c], "");
tmp = LLVMBuildBitCast(builder, tmp, ctx->i32, "");
coord = LLVMBuildInsertElement(builder, coord, tmp, index, "");
}
- emit_data->args[0] = coord;
- emit_data->output[emit_data->chan] =
- lp_build_intrinsic(builder, intr_name, emit_data->dst_type,
- emit_data->args, emit_data->arg_count,
- LP_FUNC_ATTR_READNONE | LP_FUNC_ATTR_LEGACY);
+ args->addr = coord;
}
static void build_tex_intrinsic(const struct lp_build_tgsi_action *action,
struct lp_build_tgsi_context *bld_base,
struct lp_build_emit_data *emit_data)
{
struct si_shader_context *ctx = si_shader_context(bld_base);
struct lp_build_context *base = &bld_base->base;
const struct tgsi_full_instruction *inst = emit_data->inst;
+ struct ac_image_args args;
unsigned opcode = inst->Instruction.Opcode;
unsigned target = inst->Texture.Texture;
- char intr_name[127];
- bool has_offset = inst->Texture.NumOffsets > 0;
- bool is_shadow = tgsi_is_shadow_target(target);
- char type[64];
- const char *name = "llvm.SI.image.sample";
- const char *infix = "";
if (target == TGSI_TEXTURE_BUFFER) {
emit_data->output[emit_data->chan] = lp_build_intrinsic(
base->gallivm->builder,
"llvm.SI.vs.load.input", emit_data->dst_type,
emit_data->args, emit_data->arg_count,
LP_FUNC_ATTR_READNONE | LP_FUNC_ATTR_LEGACY);
return;
}
+ memcpy(&args, emit_data->args, sizeof(args)); /* ugly */
+
+ args.opcode = ac_image_sample;
+ args.compare = tgsi_is_shadow_target(target);
+ args.offset = inst->Texture.NumOffsets > 0;
+
switch (opcode) {
case TGSI_OPCODE_TXF:
- name = target == TGSI_TEXTURE_2D_MSAA ||
- target == TGSI_TEXTURE_2D_ARRAY_MSAA ?
- "llvm.SI.image.load" :
- "llvm.SI.image.load.mip";
- is_shadow = false;
- has_offset = false;
+ args.opcode = target == TGSI_TEXTURE_2D_MSAA ||
+ target == TGSI_TEXTURE_2D_ARRAY_MSAA ?
+ ac_image_load : ac_image_load_mip;
+ args.compare = false;
+ args.offset = false;
break;
case TGSI_OPCODE_LODQ:
- name = "llvm.SI.getlod";
- is_shadow = false;
- has_offset = false;
+ args.opcode = ac_image_get_lod;
+ args.compare = false;
+ args.offset = false;
break;
case TGSI_OPCODE_TEX:
case TGSI_OPCODE_TEX2:
case TGSI_OPCODE_TXP:
if (ctx->type != PIPE_SHADER_FRAGMENT)
- infix = ".lz";
+ args.level_zero = true;
break;
case TGSI_OPCODE_TXB:
case TGSI_OPCODE_TXB2:
assert(ctx->type == PIPE_SHADER_FRAGMENT);
- infix = ".b";
+ args.bias = true;
break;
case TGSI_OPCODE_TXL:
case TGSI_OPCODE_TXL2:
- infix = ".l";
+ args.lod = true;
break;
case TGSI_OPCODE_TXD:
- infix = ".d";
+ args.deriv = true;
break;
case TGSI_OPCODE_TG4:
- name = "llvm.SI.gather4";
- infix = ".lz";
+ args.opcode = ac_image_gather4;
+ args.level_zero = true;
break;
default:
assert(0);
return;
}
- /* Add the type and suffixes .c, .o if needed. */
- ac_build_type_name_for_intr(LLVMTypeOf(emit_data->args[0]), type,
sizeof(type));
- sprintf(intr_name, "%s%s%s%s.%s",
- name, is_shadow ? ".c" : "", infix,
- has_offset ? ".o" : "", type);
-
/* The hardware needs special lowering for Gather4 with integer
formats. */
if (opcode == TGSI_OPCODE_TG4) {
struct tgsi_shader_info *info = &ctx->shader->selector->info;
/* This will also work with non-constant indexing because of how
* glsl_to_tgsi works and we intent to preserve that behavior.
*/
const unsigned src_idx = 2;
unsigned sampler = inst->Src[src_idx].Register.Index;
assert(inst->Src[src_idx].Register.File == TGSI_FILE_SAMPLER);
if (info->sampler_type[sampler] == TGSI_RETURN_TYPE_SINT ||
- info->sampler_type[sampler] == TGSI_RETURN_TYPE_UINT) {
- /* Texture coordinates start after:
- * {offset, bias, z-compare, derivatives}
- * Only the offset and z-compare can occur here.
- */
- si_lower_gather4_integer(ctx, emit_data, intr_name,
- (int)has_offset +
(int)is_shadow);
- return;
- }
+ info->sampler_type[sampler] == TGSI_RETURN_TYPE_UINT)
+ si_lower_gather4_integer(ctx, &args, target);
}
- emit_data->output[emit_data->chan] = lp_build_intrinsic(
- base->gallivm->builder, intr_name, emit_data->dst_type,
- emit_data->args, emit_data->arg_count,
- LP_FUNC_ATTR_READNONE | LP_FUNC_ATTR_LEGACY);
+ emit_data->output[emit_data->chan] =
+ ac_emit_image_opcode(&ctx->ac, &args);
}
static void si_llvm_emit_txqs(
const struct lp_build_tgsi_action *action,
struct lp_build_tgsi_context *bld_base,
struct lp_build_emit_data *emit_data)
{
struct si_shader_context *ctx = si_shader_context(bld_base);
struct gallivm_state *gallivm = bld_base->base.gallivm;
LLVMBuilderRef builder = gallivm->builder;
--
2.7.4
_______________________________________________
mesa-dev mailing list
mesa-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/mesa-dev
