From: Ben Goz <ben....@amd.com>
This patch implements the new SDMA interface functions. It also adds defines
and structures related to SDMA registers.
Signed-off-by: Ben Goz <ben.goz at amd.com>
---
drivers/gpu/drm/radeon/cik_reg.h | 200 +++++++++++++++++++++++++++++++++++-
drivers/gpu/drm/radeon/radeon_kfd.c | 132 +++++++++++++++++++++++-
2 files changed, 329 insertions(+), 3 deletions(-)
diff --git a/drivers/gpu/drm/radeon/cik_reg.h b/drivers/gpu/drm/radeon/cik_reg.h
index 79c45e8..5008964 100644
--- a/drivers/gpu/drm/radeon/cik_reg.h
+++ b/drivers/gpu/drm/radeon/cik_reg.h
@@ -147,10 +147,73 @@
#define CIK_LB_DESKTOP_HEIGHT 0x6b0c
+#define KFD_CIK_SDMA_QUEUE_OFFSET 0x200
+
+#define SQ_IND_INDEX 0x8DE0
+#define SQ_CMD 0x8DEC
+#define SQ_IND_DATA 0x8DE4
+
+#define TCP_WATCH0_ADDR_H (0x32A0*4)
+#define TCP_WATCH1_ADDR_H (0x32A3*4)
+#define TCP_WATCH2_ADDR_H (0x32A6*4)
+#define TCP_WATCH3_ADDR_H (0x32A9*4)
+#define TCP_WATCH0_ADDR_L (0x32A1*4)
+#define TCP_WATCH1_ADDR_L (0x32A4*4)
+#define TCP_WATCH2_ADDR_L (0x32A7*4)
+#define TCP_WATCH3_ADDR_L (0x32AA*4)
+#define TCP_WATCH0_CNTL (0x32A2*4)
+#define TCP_WATCH1_CNTL (0x32A5*4)
+#define TCP_WATCH2_CNTL (0x32A8*4)
+#define TCP_WATCH3_CNTL (0x32AB*4)
+
#define CP_HQD_IQ_RPTR 0xC970u
#define AQL_ENABLE (1U << 0)
-
-#define IDLE (1 << 2)
+#define SDMA0_RLC0_RB_CNTL 0xD400u
+#define RB_ENABLE (1 << 0)
+#define RB_SIZE(x) (x << 1)
+#define RB_SWAP_ENABLE (1 << 9)
+#define RPTR_WRITEBACK_ENABLE (1 << 12)
+#define RPTR_WRITEBACK_SWAP_ENABLE (1 << 13)
+#define RPTR_WRITEBACK_TIMER(x) (x << 16)
+#define RB_VMID(x) (x << 24)
+#define SDMA0_RLC0_RB_BASE 0xD404u
+#define SDMA0_RLC0_RB_BASE_HI 0xD408u
+#define SDMA0_RLC0_RB_RPTR 0xD40Cu
+#define SDMA0_RLC0_RB_WPTR 0xD410u
+#define SDMA0_RLC0_RB_WPTR_POLL_CNTL 0xD414u
+#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_HI 0xD418u
+#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_LO 0xD41Cu
+#define SDMA0_RLC0_RB_RPTR_ADDR_HI 0xD420u
+#define SDMA0_RLC0_RB_RPTR_ADDR_LO 0xD424u
+#define SDMA0_RLC0_IB_CNTL 0xD428u
+#define SDMA0_RLC0_IB_RPTR 0xD42Cu
+#define SDMA0_RLC0_IB_OFFSET 0xD430u
+#define SDMA0_RLC0_IB_BASE_LO 0xD434u
+#define SDMA0_RLC0_IB_BASE_HI 0xD438u
+#define SDMA0_RLC0_IB_SIZE 0xD43Cu
+#define SDMA0_RLC0_SKIP_CNTL 0xD440u
+#define SDMA0_RLC0_CONTEXT_STATUS 0xD444u
+#define SELECTED (1 << 0)
+#define IDLE (1 << 2)
+#define EXPIRED (1 << 3)
+#define EXCEPTION (1 << 4)
+#define CTXSW_ABLE (1 << 7)
+#define CTXSW_READY (1 << 8)
+#define SDMA0_RLC0_DOORBELL 0xD448u
+#define OFFSET(x) (x << 0)
+#define DB_ENABLE (1 << 28)
+#define CAPTURED (1 << 30)
+#define SDMA0_RLC0_VIRTUAL_ADDR 0xD49Cu
+#define ATC (1 << 0)
+#define VA_PTR32 (1 << 4)
+#define VA_SHARED_BASE(x) (x << 8)
+#define VM_HOLE (1 << 30)
+#define SDMA0_RLC0_APE1_CNTL 0xD4A0u
+#define SDMA0_RLC0_DOORBELL_LOG 0xD4A4u
+#define SDMA0_RLC0_WATERMARK 0xD4A8u
+#define SDMA0_CNTL 0xD010
+#define SDMA1_CNTL 0xD810
+#define AUTO_CTXSW_ENABLE (1 << 18)
struct cik_mqd {
uint32_t header;
@@ -283,4 +346,137 @@ struct cik_mqd {
uint32_t queue_doorbell_id15;
};
+struct cik_sdma_rlc_registers {
+ uint32_t sdma_rlc_rb_cntl;
+ uint32_t sdma_rlc_rb_base;
+ uint32_t sdma_rlc_rb_base_hi;
+ uint32_t sdma_rlc_rb_rptr;
+ uint32_t sdma_rlc_rb_wptr;
+ uint32_t sdma_rlc_rb_wptr_poll_cntl;
+ uint32_t sdma_rlc_rb_wptr_poll_addr_hi;
+ uint32_t sdma_rlc_rb_wptr_poll_addr_lo;
+ uint32_t sdma_rlc_rb_rptr_addr_hi;
+ uint32_t sdma_rlc_rb_rptr_addr_lo;
+ uint32_t sdma_rlc_ib_cntl;
+ uint32_t sdma_rlc_ib_rptr;
+ uint32_t sdma_rlc_ib_offset;
+ uint32_t sdma_rlc_ib_base_lo;
+ uint32_t sdma_rlc_ib_base_hi;
+ uint32_t sdma_rlc_ib_size;
+ uint32_t sdma_rlc_skip_cntl;
+ uint32_t sdma_rlc_context_status;
+ uint32_t sdma_rlc_doorbell;
+ uint32_t sdma_rlc_virtual_addr;
+ uint32_t sdma_rlc_ape1_cntl;
+ uint32_t sdma_rlc_doorbell_log;
+ uint32_t reserved_22;
+ uint32_t reserved_23;
+ uint32_t reserved_24;
+ uint32_t reserved_25;
+ uint32_t reserved_26;
+ uint32_t reserved_27;
+ uint32_t reserved_28;
+ uint32_t reserved_29;
+ uint32_t reserved_30;
+ uint32_t reserved_31;
+ uint32_t reserved_32;
+ uint32_t reserved_33;
+ uint32_t reserved_34;
+ uint32_t reserved_35;
+ uint32_t reserved_36;
+ uint32_t reserved_37;
+ uint32_t reserved_38;
+ uint32_t reserved_39;
+ uint32_t reserved_40;
+ uint32_t reserved_41;
+ uint32_t reserved_42;
+ uint32_t reserved_43;
+ uint32_t reserved_44;
+ uint32_t reserved_45;
+ uint32_t reserved_46;
+ uint32_t reserved_47;
+ uint32_t reserved_48;
+ uint32_t reserved_49;
+ uint32_t reserved_50;
+ uint32_t reserved_51;
+ uint32_t reserved_52;
+ uint32_t reserved_53;
+ uint32_t reserved_54;
+ uint32_t reserved_55;
+ uint32_t reserved_56;
+ uint32_t reserved_57;
+ uint32_t reserved_58;
+ uint32_t reserved_59;
+ uint32_t reserved_60;
+ uint32_t reserved_61;
+ uint32_t reserved_62;
+ uint32_t reserved_63;
+ uint32_t reserved_64;
+ uint32_t reserved_65;
+ uint32_t reserved_66;
+ uint32_t reserved_67;
+ uint32_t reserved_68;
+ uint32_t reserved_69;
+ uint32_t reserved_70;
+ uint32_t reserved_71;
+ uint32_t reserved_72;
+ uint32_t reserved_73;
+ uint32_t reserved_74;
+ uint32_t reserved_75;
+ uint32_t reserved_76;
+ uint32_t reserved_77;
+ uint32_t reserved_78;
+ uint32_t reserved_79;
+ uint32_t reserved_80;
+ uint32_t reserved_81;
+ uint32_t reserved_82;
+ uint32_t reserved_83;
+ uint32_t reserved_84;
+ uint32_t reserved_85;
+ uint32_t reserved_86;
+ uint32_t reserved_87;
+ uint32_t reserved_88;
+ uint32_t reserved_89;
+ uint32_t reserved_90;
+ uint32_t reserved_91;
+ uint32_t reserved_92;
+ uint32_t reserved_93;
+ uint32_t reserved_94;
+ uint32_t reserved_95;
+ uint32_t reserved_96;
+ uint32_t reserved_97;
+ uint32_t reserved_98;
+ uint32_t reserved_99;
+ uint32_t reserved_100;
+ uint32_t reserved_101;
+ uint32_t reserved_102;
+ uint32_t reserved_103;
+ uint32_t reserved_104;
+ uint32_t reserved_105;
+ uint32_t reserved_106;
+ uint32_t reserved_107;
+ uint32_t reserved_108;
+ uint32_t reserved_109;
+ uint32_t reserved_110;
+ uint32_t reserved_111;
+ uint32_t reserved_112;
+ uint32_t reserved_113;
+ uint32_t reserved_114;
+ uint32_t reserved_115;
+ uint32_t reserved_116;
+ uint32_t reserved_117;
+ uint32_t reserved_118;
+ uint32_t reserved_119;
+ uint32_t reserved_120;
+ uint32_t reserved_121;
+ uint32_t reserved_122;
+ uint32_t reserved_123;
+ uint32_t reserved_124;
+ uint32_t reserved_125;
+ uint32_t reserved_126;
+ uint32_t reserved_127;
+ uint32_t sdma_engine_id;
+ uint32_t sdma_queue_id;
+};
+
#endif
diff --git a/drivers/gpu/drm/radeon/radeon_kfd.c
b/drivers/gpu/drm/radeon/radeon_kfd.c
index 242fd8b..b77aee0 100644
--- a/drivers/gpu/drm/radeon/radeon_kfd.c
+++ b/drivers/gpu/drm/radeon/radeon_kfd.c
@@ -71,13 +71,17 @@ static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t
pipe_id,
static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
uint32_t queue_id, uint32_t __user *wptr);
-
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address,
uint32_t pipe_id, uint32_t queue_id);
static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
unsigned int timeout, uint32_t pipe_id,
uint32_t queue_id);
+static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
+static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+ unsigned int timeout);
+static int kgd_init_sdma_engines(struct kgd_dev *kgd);
static const struct kfd2kgd_calls kfd2kgd = {
.init_sa_manager = init_sa_manager,
@@ -91,9 +95,13 @@ static const struct kfd2kgd_calls kfd2kgd = {
.set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
.init_memory = kgd_init_memory,
.init_pipeline = kgd_init_pipeline,
+ .init_sdma_engines = kgd_init_sdma_engines,
.hqd_load = kgd_hqd_load,
+ .hqd_sdma_load = kgd_hqd_sdma_load,
.hqd_is_occupies = kgd_hqd_is_occupies,
+ .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
.hqd_destroy = kgd_hqd_destroy,
+ .hqd_sdma_destroy = kgd_hqd_sdma_destroy,
.get_fw_version = get_fw_version
};
@@ -435,11 +443,43 @@ static int kgd_init_pipeline(struct kgd_dev *kgd,
uint32_t pipe_id,
return 0;
}
+static int kgd_init_sdma_engines(struct kgd_dev *kgd)
+{
+ uint32_t value;
+
+ value = read_register(kgd, SDMA0_CNTL);
+ value |= AUTO_CTXSW_ENABLE;
+ write_register(kgd, SDMA0_CNTL, value);
+
+ value = read_register(kgd, SDMA1_CNTL);
+ value |= AUTO_CTXSW_ENABLE;
+ write_register(kgd, SDMA1_CNTL, value);
+
+ return 0;
+}
+
+static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m)
+{
+ uint32_t retval;
+
+ retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET +
+ m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET;
+
+ pr_debug("kfd: sdma base address: 0x%x\n", retval);
+
+ return retval;
+}
+
static inline struct cik_mqd *get_mqd(void *mqd)
{
return (struct cik_mqd *)mqd;
}
+static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
+{
+ return (struct cik_sdma_rlc_registers *)mqd;
+}
+
static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
uint32_t queue_id, uint32_t __user *wptr)
{
@@ -517,6 +557,45 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd,
uint32_t pipe_id,
return 0;
}
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
+{
+ struct cik_sdma_rlc_registers *m;
+ uint32_t sdma_base_addr;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(m);
+
+ write_register(kgd,
+ sdma_base_addr + SDMA0_RLC0_VIRTUAL_ADDR,
+ m->sdma_rlc_virtual_addr);
+
+ write_register(kgd,
+ sdma_base_addr + SDMA0_RLC0_RB_BASE,
+ m->sdma_rlc_rb_base);
+
+ write_register(kgd,
+ sdma_base_addr + SDMA0_RLC0_RB_BASE_HI,
+ m->sdma_rlc_rb_base_hi);
+
+ write_register(kgd,
+ sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_LO,
+ m->sdma_rlc_rb_rptr_addr_lo);
+
+ write_register(kgd,
+ sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_HI,
+ m->sdma_rlc_rb_rptr_addr_hi);
+
+ write_register(kgd,
+ sdma_base_addr + SDMA0_RLC0_DOORBELL,
+ m->sdma_rlc_doorbell);
+
+ write_register(kgd,
+ sdma_base_addr + SDMA0_RLC0_RB_CNTL,
+ m->sdma_rlc_rb_cntl);
+
+ return 0;
+}
+
static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address,
uint32_t pipe_id, uint32_t queue_id)
{
@@ -538,6 +617,24 @@ static bool kgd_hqd_is_occupies(struct kgd_dev *kgd,
uint64_t queue_address,
return retval;
}
+static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
+{
+ struct cik_sdma_rlc_registers *m;
+ uint32_t sdma_base_addr;
+ uint32_t sdma_rlc_rb_cntl;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(m);
+
+ sdma_rlc_rb_cntl = read_register(kgd,
+ sdma_base_addr + SDMA0_RLC0_RB_CNTL);
+
+ if (sdma_rlc_rb_cntl & RB_ENABLE)
+ return true;
+
+ return false;
+}
+
static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
unsigned int timeout, uint32_t pipe_id,
uint32_t queue_id)
@@ -566,6 +663,39 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t
reset_type,
return 0;
}
+static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+ unsigned int timeout)
+{
+ struct cik_sdma_rlc_registers *m;
+ uint32_t sdma_base_addr;
+ uint32_t temp;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(m);
+
+ temp = read_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL);
+ temp = temp & ~RB_ENABLE;
+ write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL, temp);
+
+ while (true) {
+ temp = read_register(kgd, sdma_base_addr +
+ SDMA0_RLC0_CONTEXT_STATUS);
+ if (temp & IDLE)
+ break;
+ if (timeout == 0)
+ return -ETIME;
+ msleep(20);
+ timeout -= 20;
+ }
+
+ write_register(kgd, sdma_base_addr + SDMA0_RLC0_DOORBELL, 0);
+ write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_RPTR, 0);
+ write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_WPTR, 0);
+ write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_BASE, 0);
+
+ return 0;
+}
+
static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
{
struct radeon_device *rdev = (struct radeon_device *) kgd;
--
1.9.1
