From: Prem Mallappa <prem.malla...@broadcom.com> Introduces the SMMUv3 derived model. This is based on System MMUv3 specification (v17).
Signed-off-by: Prem Mallappa <prem.malla...@broadcom.com> Signed-off-by: Eric Auger <eric.au...@redhat.com> --- v3 -> v4 - smmu_irq_update - fix hash key allocation - set smmu_iommu_ops - set SMMU_REG_CR0, - smmuv3_translate: ret.perm not set in bypass mode - use trace events - renamed STM2U64 into L1STD_L2PTR and STMSPAN into L1STD_SPAN - rework smmu_find_ste - fix tg2granule in TT0/0b10 corresponds to 16kB v2 -> v3: - move creation of include/hw/arm/smmuv3.h to this patch to fix compil issue - compilation allowed - fix sbus allocation in smmu_init_pci_iommu - restructure code into headers - misc cleanups --- hw/arm/Makefile.objs | 2 +- hw/arm/smmuv3-internal.h | 603 ++++++++++++++++++++++++ hw/arm/smmuv3.c | 1134 ++++++++++++++++++++++++++++++++++++++++++++++ hw/arm/trace-events | 32 ++ include/hw/arm/smmuv3.h | 87 ++++ 5 files changed, 1857 insertions(+), 1 deletion(-) create mode 100644 hw/arm/smmuv3-internal.h create mode 100644 hw/arm/smmuv3.c create mode 100644 include/hw/arm/smmuv3.h diff --git a/hw/arm/Makefile.objs b/hw/arm/Makefile.objs index 6c7d4af..02cd23f 100644 --- a/hw/arm/Makefile.objs +++ b/hw/arm/Makefile.objs @@ -18,4 +18,4 @@ obj-$(CONFIG_FSL_IMX25) += fsl-imx25.o imx25_pdk.o obj-$(CONFIG_FSL_IMX31) += fsl-imx31.o kzm.o obj-$(CONFIG_FSL_IMX6) += fsl-imx6.o sabrelite.o obj-$(CONFIG_ASPEED_SOC) += aspeed_soc.o aspeed.o -obj-$(CONFIG_ARM_SMMUV3) += smmu-common.o +obj-$(CONFIG_ARM_SMMUV3) += smmu-common.o smmuv3.o diff --git a/hw/arm/smmuv3-internal.h b/hw/arm/smmuv3-internal.h new file mode 100644 index 0000000..db0048f --- /dev/null +++ b/hw/arm/smmuv3-internal.h @@ -0,0 +1,603 @@ +/* + * ARM SMMUv3 support - Internal API + * + * Copyright (C) 2014-2016 Broadcom Corporation + * Copyright (c) 2017 Red Hat, Inc. + * Written by Prem Mallappa, Eric Auger + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef HW_ARM_SMMU_V3_INTERNAL_H +#define HW_ARM_SMMU_V3_INTERNAL_H + +#include "trace.h" +#include "qemu/error-report.h" +#include "hw/arm/smmu-common.h" + +/***************************** + * MMIO Register + *****************************/ +enum { + SMMU_REG_IDR0 = 0x0, + +/* IDR0 Field Values and supported features */ + +#define SMMU_IDR0_S2P 1 /* stage 2 */ +#define SMMU_IDR0_S1P 1 /* stage 1 */ +#define SMMU_IDR0_TTF 2 /* Aarch64 only - not Aarch32 (LPAE) */ +#define SMMU_IDR0_COHACC 1 /* IO coherent access */ +#define SMMU_IDR0_HTTU 2 /* Access and Dirty flag update */ +#define SMMU_IDR0_HYP 0 /* Hypervisor Stage 1 contexts */ +#define SMMU_IDR0_ATS 0 /* PCIe RC ATS */ +#define SMMU_IDR0_ASID16 1 /* 16-bit ASID */ +#define SMMU_IDR0_PRI 0 /* Page Request Interface */ +#define SMMU_IDR0_VMID16 0 /* 16-bit VMID */ +#define SMMU_IDR0_CD2L 0 /* 2-level Context Descriptor table */ +#define SMMU_IDR0_STALL 1 /* Stalling fault model */ +#define SMMU_IDR0_TERM 1 /* Termination model behaviour */ +#define SMMU_IDR0_STLEVEL 1 /* Multi-level Stream Table */ + +#define SMMU_IDR0_S2P_SHIFT 0 +#define SMMU_IDR0_S1P_SHIFT 1 +#define SMMU_IDR0_TTF_SHIFT 2 +#define SMMU_IDR0_COHACC_SHIFT 4 +#define SMMU_IDR0_HTTU_SHIFT 6 +#define SMMU_IDR0_HYP_SHIFT 9 +#define SMMU_IDR0_ATS_SHIFT 10 +#define SMMU_IDR0_ASID16_SHIFT 12 +#define SMMU_IDR0_PRI_SHIFT 16 +#define SMMU_IDR0_VMID16_SHIFT 18 +#define SMMU_IDR0_CD2L_SHIFT 19 +#define SMMU_IDR0_STALL_SHIFT 24 +#define SMMU_IDR0_TERM_SHIFT 26 +#define SMMU_IDR0_STLEVEL_SHIFT 27 + + SMMU_REG_IDR1 = 0x4, +#define SMMU_IDR1_SIDSIZE 16 + SMMU_REG_IDR2 = 0x8, + SMMU_REG_IDR3 = 0xc, + SMMU_REG_IDR4 = 0x10, + SMMU_REG_IDR5 = 0x14, +#define SMMU_IDR5_GRAN_SHIFT 4 +#define SMMU_IDR5_GRAN 0b101 /* GRAN4K, GRAN64K */ +#define SMMU_IDR5_OAS 4 /* 44 bits */ + SMMU_REG_IIDR = 0x1c, + SMMU_REG_CR0 = 0x20, + +#define SMMU_CR0_SMMU_ENABLE (1 << 0) +#define SMMU_CR0_PRIQ_ENABLE (1 << 1) +#define SMMU_CR0_EVTQ_ENABLE (1 << 2) +#define SMMU_CR0_CMDQ_ENABLE (1 << 3) +#define SMMU_CR0_ATS_CHECK (1 << 4) + + SMMU_REG_CR0_ACK = 0x24, + SMMU_REG_CR1 = 0x28, + SMMU_REG_CR2 = 0x2c, + + SMMU_REG_STATUSR = 0x40, + + SMMU_REG_IRQ_CTRL = 0x50, + SMMU_REG_IRQ_CTRL_ACK = 0x54, + +#define SMMU_IRQ_CTRL_GERROR_EN (1 << 0) +#define SMMU_IRQ_CTRL_EVENT_EN (1 << 1) +#define SMMU_IRQ_CTRL_PRI_EN (1 << 2) + + SMMU_REG_GERROR = 0x60, + +#define SMMU_GERROR_CMDQ (1 << 0) +#define SMMU_GERROR_EVENTQ (1 << 2) +#define SMMU_GERROR_PRIQ (1 << 3) +#define SMMU_GERROR_MSI_CMDQ (1 << 4) +#define SMMU_GERROR_MSI_EVENTQ (1 << 5) +#define SMMU_GERROR_MSI_PRIQ (1 << 6) +#define SMMU_GERROR_MSI_GERROR (1 << 7) +#define SMMU_GERROR_SFM_ERR (1 << 8) + + SMMU_REG_GERRORN = 0x64, + SMMU_REG_GERROR_IRQ_CFG0 = 0x68, + SMMU_REG_GERROR_IRQ_CFG1 = 0x70, + SMMU_REG_GERROR_IRQ_CFG2 = 0x74, + + /* SMMU_BASE_RA Applies to STRTAB_BASE, CMDQ_BASE and EVTQ_BASE */ +#define SMMU_BASE_RA (1ULL << 62) + SMMU_REG_STRTAB_BASE = 0x80, + SMMU_REG_STRTAB_BASE_CFG = 0x88, + + SMMU_REG_CMDQ_BASE = 0x90, + SMMU_REG_CMDQ_PROD = 0x98, + SMMU_REG_CMDQ_CONS = 0x9c, + /* CMD Consumer (CONS) */ +#define SMMU_CMD_CONS_ERR_SHIFT 24 +#define SMMU_CMD_CONS_ERR_BITS 7 + + SMMU_REG_EVTQ_BASE = 0xa0, + SMMU_REG_EVTQ_PROD = 0xa8, + SMMU_REG_EVTQ_CONS = 0xac, + SMMU_REG_EVTQ_IRQ_CFG0 = 0xb0, + SMMU_REG_EVTQ_IRQ_CFG1 = 0xb8, + SMMU_REG_EVTQ_IRQ_CFG2 = 0xbc, + + SMMU_REG_PRIQ_BASE = 0xc0, + SMMU_REG_PRIQ_PROD = 0xc8, + SMMU_REG_PRIQ_CONS = 0xcc, + SMMU_REG_PRIQ_IRQ_CFG0 = 0xd0, + SMMU_REG_PRIQ_IRQ_CFG1 = 0xd8, + SMMU_REG_PRIQ_IRQ_CFG2 = 0xdc, + + SMMU_ID_REGS_OFFSET = 0xfd0, + + /* Secure registers are not used for now */ + SMMU_SECURE_OFFSET = 0x8000, +}; + +/********************** + * Data Structures + **********************/ + +struct __smmu_data2 { + uint32_t word[2]; +}; + +struct __smmu_data8 { + uint32_t word[8]; +}; + +struct __smmu_data16 { + uint32_t word[16]; +}; + +struct __smmu_data4 { + uint32_t word[4]; +}; + +typedef struct __smmu_data2 STEDesc; /* STE Level 1 Descriptor */ +typedef struct __smmu_data16 Ste; /* Stream Table Entry(STE) */ +typedef struct __smmu_data2 CDDesc; /* CD Level 1 Descriptor */ +typedef struct __smmu_data16 Cd; /* Context Descriptor(CD) */ + +typedef struct __smmu_data4 Cmd; /* Command Entry */ +typedef struct __smmu_data8 Evt; /* Event Entry */ +typedef struct __smmu_data4 Pri; /* PRI entry */ + +/***************************** + * STE fields + *****************************/ + +#define STE_VALID(x) extract32((x)->word[0], 0, 1) /* 0 */ +#define STE_CONFIG(x) extract32((x)->word[0], 1, 3) +enum { + STE_CONFIG_NONE = 0, + STE_CONFIG_BYPASS = 4, /* S1 Bypass, S2 Bypass */ + STE_CONFIG_S1TR = 1, /* S1 Translate, S2 Bypass */ + STE_CONFIG_S2TR = 2, /* S1 Bypass, S2 Translate */ + STE_CONFIG_S1TR_S2TR = 3, /* S1 Translate, S2 Translate */ +}; +#define STE_S1FMT(x) extract32((x)->word[0], 4, 2) +#define STE_S1CDMAX(x) extract32((x)->word[1], 27, 5) +#define STE_EATS(x) extract32((x)->word[2], 28, 2) +#define STE_STRW(x) extract32((x)->word[2], 30, 2) +#define STE_S2VMID(x) extract32((x)->word[4], 0, 16) +#define STE_S2T0SZ(x) extract32((x)->word[5], 0, 6) +#define STE_S2TG(x) extract32((x)->word[5], 14, 2) +#define STE_S2PS(x) extract32((x)->word[5], 16, 3) +#define STE_S2AA64(x) extract32((x)->word[5], 19, 1) +#define STE_S2HD(x) extract32((x)->word[5], 24, 1) +#define STE_S2HA(x) extract32((x)->word[5], 25, 1) +#define STE_S2S(x) extract32((x)->word[5], 26, 1) +#define STE_CTXPTR(x) \ + ({ \ + unsigned long addr; \ + addr = (uint64_t)extract32((x)->word[1], 0, 16) << 32; \ + addr |= (uint64_t)((x)->word[0] & 0xffffffc0); \ + addr; \ + }) + +#define STE_S2TTB(x) \ + ({ \ + unsigned long addr; \ + addr = (uint64_t)extract32((x)->word[7], 0, 16) << 32; \ + addr |= (uint64_t)((x)->word[6] & 0xfffffff0); \ + addr; \ + }) + +static inline int is_ste_bypass(Ste *ste) +{ + return STE_CONFIG(ste) == STE_CONFIG_BYPASS; +} + +static inline bool has_stage1(Ste *ste) +{ + return STE_CONFIG(ste) & 0b001; +} + +static inline bool has_stage2(Ste *ste) +{ + return STE_CONFIG(ste) & 0b010; +} + +/** + * is_s2granule_valid - Check the stage 2 translation granule size + * advertised in the STE matches any IDR5 supported value + */ +static inline bool is_s2granule_valid(Ste *ste) +{ + int idr5_format = 0; + + switch (STE_S2TG(ste)) { + case 0: /* 4kB */ + idr5_format = 0x1; + break; + case 1: /* 64 kB */ + idr5_format = 0x4; + break; + case 2: /* 16 kB */ + idr5_format = 0x2; + break; + case 3: /* reserved */ + break; + } + idr5_format &= SMMU_IDR5_GRAN; + return idr5_format; +} + +static inline int oas2bits(int oas_field) +{ + switch (oas_field) { + case 0b011: + return 42; + case 0b100: + return 44; + default: + return 32 + (1 << oas_field); + } +} + +static inline int pa_range(Ste *ste) +{ + int oas_field = MIN(STE_S2PS(ste), SMMU_IDR5_OAS); + + if (!STE_S2AA64(ste)) { + return 40; + } + + return oas2bits(oas_field); +} + +#define MAX_PA(ste) ((1 << pa_range(ste)) - 1) + +/***************************** + * CD fields + *****************************/ +#define CD_VALID(x) extract32((x)->word[0], 30, 1) +#define CD_ASID(x) extract32((x)->word[1], 16, 16) +#define CD_TTB(x, sel) \ + ({ \ + uint64_t hi, lo; \ + hi = extract32((x)->word[(sel) * 2 + 3], 0, 16); \ + hi <<= 32; \ + lo = (x)->word[(sel) * 2 + 2] & ~0xf; \ + hi | lo; \ + }) + +#define CD_TSZ(x, sel) extract32((x)->word[0], (16 * (sel)) + 0, 6) +#define CD_TG(x, sel) extract32((x)->word[0], (16 * (sel)) + 6, 2) +#define CD_EPD(x, sel) extract32((x)->word[0], (16 * (sel)) + 14, 1) + +#define CD_T0SZ(x) CD_TSZ((x), 0) +#define CD_T1SZ(x) CD_TSZ((x), 1) +#define CD_TG0(x) CD_TG((x), 0) +#define CD_TG1(x) CD_TG((x), 1) +#define CD_EPD0(x) CD_EPD((x), 0) +#define CD_EPD1(x) CD_EPD((x), 1) +#define CD_IPS(x) extract32((x)->word[1], 0, 3) +#define CD_AARCH64(x) extract32((x)->word[1], 9, 1) +#define CD_TTB0(x) CD_TTB((x), 0) +#define CD_TTB1(x) CD_TTB((x), 1) + +#define CDM_VALID(x) ((x)->word[0] & 0x1) + +static inline int is_cd_valid(SMMUV3State *s, Ste *ste, Cd *cd) +{ + return CD_VALID(cd); +} + +/***************************** + * Commands + *****************************/ +enum { + SMMU_CMD_PREFETCH_CONFIG = 0x01, + SMMU_CMD_PREFETCH_ADDR, + SMMU_CMD_CFGI_STE, + SMMU_CMD_CFGI_STE_RANGE, + SMMU_CMD_CFGI_CD, + SMMU_CMD_CFGI_CD_ALL, + SMMU_CMD_CFGI_ALL, + SMMU_CMD_TLBI_NH_ALL = 0x10, + SMMU_CMD_TLBI_NH_ASID, + SMMU_CMD_TLBI_NH_VA, + SMMU_CMD_TLBI_NH_VAA, + SMMU_CMD_TLBI_EL3_ALL = 0x18, + SMMU_CMD_TLBI_EL3_VA = 0x1a, + SMMU_CMD_TLBI_EL2_ALL = 0x20, + SMMU_CMD_TLBI_EL2_ASID, + SMMU_CMD_TLBI_EL2_VA, + SMMU_CMD_TLBI_EL2_VAA, /* 0x23 */ + SMMU_CMD_TLBI_S12_VMALL = 0x28, + SMMU_CMD_TLBI_S2_IPA = 0x2a, + SMMU_CMD_TLBI_NSNH_ALL = 0x30, + SMMU_CMD_ATC_INV = 0x40, + SMMU_CMD_PRI_RESP, + SMMU_CMD_RESUME = 0x44, + SMMU_CMD_STALL_TERM, + SMMU_CMD_SYNC, /* 0x46 */ +}; + +/***************************** + * Register Access Primitives + *****************************/ + +static inline void smmu_write64_reg(SMMUV3State *s, uint32_t addr, uint64_t val) +{ + addr >>= 2; + s->regs[addr] = val & 0xFFFFFFFFULL; + s->regs[addr + 1] = val & ~0xFFFFFFFFULL; +} + +static inline void smmu_write_reg(SMMUV3State *s, uint32_t addr, uint64_t val) +{ + s->regs[addr >> 2] = val; +} + +static inline uint32_t smmu_read_reg(SMMUV3State *s, uint32_t addr) +{ + return s->regs[addr >> 2]; +} + +static inline uint64_t smmu_read64_reg(SMMUV3State *s, uint32_t addr) +{ + addr >>= 2; + return s->regs[addr] | (s->regs[addr + 1] << 32); +} + +#define smmu_read32_reg smmu_read_reg +#define smmu_write32_reg smmu_write_reg + +/***************************** + * CMDQ fields + *****************************/ + +enum { /* Command Errors */ + SMMU_CMD_ERR_NONE = 0, + SMMU_CMD_ERR_ILLEGAL, + SMMU_CMD_ERR_ABORT +}; + +enum { /* Command completion notification */ + CMD_SYNC_SIG_NONE, + CMD_SYNC_SIG_IRQ, + CMD_SYNC_SIG_SEV, +}; + +#define CMD_TYPE(x) extract32((x)->word[0], 0, 8) +#define CMD_SEC(x) extract32((x)->word[0], 9, 1) +#define CMD_SEV(x) extract32((x)->word[0], 10, 1) +#define CMD_AC(x) extract32((x)->word[0], 12, 1) +#define CMD_AB(x) extract32((x)->word[0], 13, 1) +#define CMD_CS(x) extract32((x)->word[0], 12, 2) +#define CMD_SSID(x) extract32((x)->word[0], 16, 16) +#define CMD_SID(x) ((x)->word[1]) +#define CMD_VMID(x) extract32((x)->word[1], 0, 16) +#define CMD_ASID(x) extract32((x)->word[1], 16, 16) +#define CMD_STAG(x) extract32((x)->word[2], 0, 16) +#define CMD_RESP(x) extract32((x)->word[2], 11, 2) +#define CMD_GRPID(x) extract32((x)->word[3], 0, 8) +#define CMD_SIZE(x) extract32((x)->word[3], 0, 16) +#define CMD_LEAF(x) extract32((x)->word[3], 0, 1) +#define CMD_SPAN(x) extract32((x)->word[3], 0, 5) +#define CMD_ADDR(x) ({ \ + uint64_t addr = (uint64_t)(x)->word[3]; \ + addr <<= 32; \ + addr |= extract32((x)->word[3], 12, 20); \ + addr; \ + }) + +/*************************** + * Queue Handling + ***************************/ + +typedef enum { + CMD_Q_EMPTY, + CMD_Q_FULL, + CMD_Q_INUSE, +} SMMUQStatus; + +#define Q_ENTRY(q, idx) (q->base + q->ent_size * idx) +#define Q_WRAP(q, pc) ((pc) >> (q)->shift) +#define Q_IDX(q, pc) ((pc) & ((1 << (q)->shift) - 1)) + +static inline SMMUQStatus +__smmu_queue_status(SMMUV3State *s, SMMUQueue *q) +{ + uint32_t prod = Q_IDX(q, q->prod), cons = Q_IDX(q, q->cons); + if ((prod == cons) && (q->wrap.prod != q->wrap.cons)) { + return CMD_Q_FULL; + } else if ((prod == cons) && (q->wrap.prod == q->wrap.cons)) { + return CMD_Q_EMPTY; + } + return CMD_Q_INUSE; +} +#define smmu_is_q_full(s, q) (__smmu_queue_status(s, q) == CMD_Q_FULL) +#define smmu_is_q_empty(s, q) (__smmu_queue_status(s, q) == CMD_Q_EMPTY) + +static inline int __smmu_q_enabled(SMMUV3State *s, uint32_t q) +{ + return smmu_read32_reg(s, SMMU_REG_CR0) & q; +} +#define smmu_cmd_q_enabled(s) __smmu_q_enabled(s, SMMU_CR0_CMDQ_ENABLE) +#define smmu_evt_q_enabled(s) __smmu_q_enabled(s, SMMU_CR0_EVTQ_ENABLE) + +#define SMMU_CMDQ_ERR(s) ((smmu_read32_reg(s, SMMU_REG_GERROR) ^ \ + smmu_read32_reg(s, SMMU_REG_GERRORN)) & \ + SMMU_GERROR_CMDQ) + +/***************************** + * EVTQ fields + *****************************/ + +#define EVT_Q_OVERFLOW (1 << 31) + +#define EVT_SET_TYPE(x, t) deposit32((x)->word[0], 0, 8, t) +#define EVT_SET_SID(x, s) ((x)->word[1] = s) +#define EVT_SET_INPUT_ADDR(x, addr) ({ \ + (x)->word[5] = (uint32_t)(addr >> 32); \ + (x)->word[4] = (uint32_t)(addr & 0xffffffff); \ + addr; \ + }) + +/***************************** + * Events + *****************************/ + +enum evt_err { + SMMU_EVT_F_UUT = 0x1, + SMMU_EVT_C_BAD_SID, + SMMU_EVT_F_STE_FETCH, + SMMU_EVT_C_BAD_STE, + SMMU_EVT_F_BAD_ATS_REQ, + SMMU_EVT_F_STREAM_DISABLED, + SMMU_EVT_F_TRANS_FORBIDDEN, + SMMU_EVT_C_BAD_SSID, + SMMU_EVT_F_CD_FETCH, + SMMU_EVT_C_BAD_CD, + SMMU_EVT_F_WALK_EXT_ABRT, + SMMU_EVT_F_TRANS = 0x10, + SMMU_EVT_F_ADDR_SZ, + SMMU_EVT_F_ACCESS, + SMMU_EVT_F_PERM, + SMMU_EVT_F_TLB_CONFLICT = 0x20, + SMMU_EVT_F_CFG_CONFLICT = 0x21, + SMMU_EVT_E_PAGE_REQ = 0x24, +}; + +typedef enum evt_err SMMUEvtErr; + +/***************************** + * Interrupts + *****************************/ + +static inline int __smmu_irq_enabled(SMMUV3State *s, uint32_t q) +{ + return smmu_read64_reg(s, SMMU_REG_IRQ_CTRL) & q; +} +#define smmu_evt_irq_enabled(s) \ + __smmu_irq_enabled(s, SMMU_IRQ_CTRL_EVENT_EN) +#define smmu_gerror_irq_enabled(s) \ + __smmu_irq_enabled(s, SMMU_IRQ_CTRL_GERROR_EN) +#define smmu_pri_irq_enabled(s) \ + __smmu_irq_enabled(s, SMMU_IRQ_CTRL_PRI_EN) + +static inline bool +smmu_is_irq_pending(SMMUV3State *s, int irq) +{ + return smmu_read32_reg(s, SMMU_REG_GERROR) ^ + smmu_read32_reg(s, SMMU_REG_GERRORN); +} + +/***************************** + * Hash Table + *****************************/ + +static inline gboolean smmu_uint64_equal(gconstpointer v1, gconstpointer v2) +{ + return *((const uint64_t *)v1) == *((const uint64_t *)v2); +} + +static inline guint smmu_uint64_hash(gconstpointer v) +{ + return (guint)*(const uint64_t *)v; +} + +/***************************** + * Misc + *****************************/ + +/** + * tg2granule - Decodes the CD translation granule size field according + * to the TT in use + * @bits: TG0/1 fiels + * @tg1: if set, @bits belong to TG1, otherwise belong to TG0 + */ +static inline int tg2granule(int bits, bool tg1) +{ + switch (bits) { + case 1: + return tg1 ? 14 : 16; + case 2: + return tg1 ? 12 : 14; + case 3: + return tg1 ? 16 : 12; + default: + return 12; + } +} + +#define L1STD_L2PTR(stm) ({ \ + uint64_t hi, lo; \ + hi = (stm)->word[1]; \ + lo = (stm)->word[0] & ~(uint64_t)0x1f; \ + hi << 32 | lo; \ + }) + +#define L1STD_SPAN(stm) (extract32((stm)->word[0], 0, 4)) + +/***************************** + * Debug + *****************************/ +/* #define ARM_SMMU_DEBUG */ + +#ifdef ARM_SMMU_DEBUG +static inline void dump_ste(Ste *ste) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(ste->word); i += 2) { + trace_smmu_dump_ste(i, ste->word[i], i + 1, ste->word[i + 1]); + } +} + +static inline void dump_cd(Cd *cd) +{ + int i; + for (i = 0; i < ARRAY_SIZE(cd->word); i += 2) { + trace_smmu_dump_cd(i, cd->word[i], i + 1, cd->word[i + 1]); + } +} + +static inline void dump_cmd(Cmd *cmd) +{ + int i; + for (i = 0; i < ARRAY_SIZE(cmd->word); i += 2) { + trace_smmu_dump_cmd(i, cmd->word[i], i + 1, cmd->word[i + 1]); + } +} + +#else +#define dump_ste(...) do {} while (0) +#define dump_cd(...) do {} while (0) +#define dump_cmd(...) do {} while (0) +#endif /* ARM_SMMU_DEBUG */ + +#endif diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c new file mode 100644 index 0000000..aa6e991 --- /dev/null +++ b/hw/arm/smmuv3.c @@ -0,0 +1,1134 @@ +/* + * Copyright (C) 2014-2016 Broadcom Corporation + * Copyright (c) 2017 Red Hat, Inc. + * Written by Prem Mallappa, Eric Auger + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "qemu/osdep.h" +#include "hw/boards.h" +#include "sysemu/sysemu.h" +#include "hw/sysbus.h" +#include "hw/pci/pci.h" +#include "exec/address-spaces.h" +#include "trace.h" +#include "qemu/error-report.h" + +#include "hw/arm/smmuv3.h" +#include "smmuv3-internal.h" + +static inline int smmu_enabled(SMMUV3State *s) +{ + return smmu_read32_reg(s, SMMU_REG_CR0) & SMMU_CR0_SMMU_ENABLE; +} + +/** + * smmu_irq_update - update the GERROR register according to + * the IRQ and the enable state + * + * return > 0 when IRQ is supposed to be raised + */ +static int smmu_irq_update(SMMUV3State *s, int irq, uint64_t data) +{ + uint32_t error = 0; + + if (!smmu_gerror_irq_enabled(s)) { + return 0; + } + + switch (irq) { + case SMMU_IRQ_EVTQ: + if (smmu_evt_irq_enabled(s)) { + error = SMMU_GERROR_EVENTQ; + } + break; + case SMMU_IRQ_CMD_SYNC: + if (smmu_gerror_irq_enabled(s)) { + uint32_t err_type = (uint32_t)data; + + if (err_type) { + uint32_t regval = smmu_read32_reg(s, SMMU_REG_CMDQ_CONS); + smmu_write32_reg(s, SMMU_REG_CMDQ_CONS, + regval | err_type << SMMU_CMD_CONS_ERR_SHIFT); + } + error = SMMU_GERROR_CMDQ; + } + break; + case SMMU_IRQ_PRIQ: + if (smmu_pri_irq_enabled(s)) { + error = SMMU_GERROR_PRIQ; + } + break; + } + + if (error) { + uint32_t gerror = smmu_read32_reg(s, SMMU_REG_GERROR); + uint32_t gerrorn = smmu_read32_reg(s, SMMU_REG_GERRORN); + + trace_smmuv3_irq_update(error, gerror, gerrorn); + + /* only toggle GERROR if the interrupt is not active */ + if (!((gerror ^ gerrorn) & error)) { + smmu_write32_reg(s, SMMU_REG_GERROR, gerror ^ error); + } + } + + return error; +} + +static void smmu_irq_raise(SMMUV3State *s, int irq, uint64_t data) +{ + trace_smmuv3_irq_raise(irq); + if (smmu_irq_update(s, irq, data)) { + qemu_irq_raise(s->irq[irq]); + } +} + +static MemTxResult smmu_q_read(SMMUV3State *s, SMMUQueue *q, void *data) +{ + uint64_t addr = Q_ENTRY(q, Q_IDX(q, q->cons)); + + q->cons++; + if (q->cons == q->entries) { + q->cons = 0; + q->wrap.cons++; /* this will toggle */ + } + + return smmu_read_sysmem(addr, data, q->ent_size, false); +} + +static MemTxResult smmu_q_write(SMMUV3State *s, SMMUQueue *q, void *data) +{ + uint64_t addr = Q_ENTRY(q, Q_IDX(q, q->prod)); + + if (q->prod == q->entries) { + q->prod = 0; + q->wrap.prod++; /* this will toggle */ + } + + q->prod++; + + smmu_write_sysmem(addr, data, q->ent_size, false); + + return MEMTX_OK; +} + +static MemTxResult smmu_read_cmdq(SMMUV3State *s, Cmd *cmd) +{ + SMMUQueue *q = &s->cmdq; + MemTxResult ret = smmu_q_read(s, q, cmd); + uint32_t val = 0; + + val |= (q->wrap.cons << q->shift) | q->cons; + + /* Update consumer pointer */ + smmu_write32_reg(s, SMMU_REG_CMDQ_CONS, val); + + return ret; +} + +static int smmu_cmdq_consume(SMMUV3State *s) +{ + uint32_t error = SMMU_CMD_ERR_NONE; + + trace_smmuv3_cmdq_consume(SMMU_CMDQ_ERR(s)); + + if (!smmu_cmd_q_enabled(s)) { + return 0; + } + + while (!SMMU_CMDQ_ERR(s) && !smmu_is_q_empty(s, &s->cmdq)) { + SMMUQueue *q = &s->cmdq; + uint32_t type; + Cmd cmd; + + if (smmu_read_cmdq(s, &cmd) != MEMTX_OK) { + error = SMMU_CMD_ERR_ABORT; + break; + } + + trace_smmuv3_cmdq_consume_details(q->base, q->cons, q->prod, + cmd.word[0], q->wrap.cons); + + type = CMD_TYPE(&cmd); + + switch (CMD_TYPE(&cmd)) { + case SMMU_CMD_SYNC: /* Fallthrough */ + if (CMD_CS(&cmd) & CMD_SYNC_SIG_IRQ) { + smmu_irq_raise(s, SMMU_IRQ_CMD_SYNC, SMMU_CMD_ERR_NONE); + } else if (CMD_CS(&cmd) & CMD_SYNC_SIG_SEV) { + trace_smmuv3_cmdq_consume_sev(); + } + break; + case SMMU_CMD_PREFETCH_CONFIG: + case SMMU_CMD_PREFETCH_ADDR: + case SMMU_CMD_CFGI_STE: + case SMMU_CMD_CFGI_STE_RANGE: /* same as SMMU_CMD_CFGI_ALL */ + case SMMU_CMD_CFGI_CD: + case SMMU_CMD_CFGI_CD_ALL: + case SMMU_CMD_TLBI_NH_ALL: + case SMMU_CMD_TLBI_NH_ASID: + case SMMU_CMD_TLBI_NH_VA: + case SMMU_CMD_TLBI_NH_VAA: + case SMMU_CMD_TLBI_EL3_ALL: + case SMMU_CMD_TLBI_EL3_VA: + case SMMU_CMD_TLBI_EL2_ALL: + case SMMU_CMD_TLBI_EL2_ASID: + case SMMU_CMD_TLBI_EL2_VA: + case SMMU_CMD_TLBI_EL2_VAA: + case SMMU_CMD_TLBI_S12_VMALL: + case SMMU_CMD_TLBI_S2_IPA: + case SMMU_CMD_TLBI_NSNH_ALL: + case SMMU_CMD_ATC_INV: + case SMMU_CMD_PRI_RESP: + case SMMU_CMD_RESUME: + case SMMU_CMD_STALL_TERM: + trace_smmuv3_unhandled_cmd(type); + break; + default: + error = SMMU_CMD_ERR_ILLEGAL; + error_report("Illegal command type: %d, ignoring", CMD_TYPE(&cmd)); + dump_cmd(&cmd); + break; + } + + if (error != SMMU_CMD_ERR_NONE) { + error_report("CMD Error"); + break; + } + } + + if (error) { + smmu_irq_raise(s, SMMU_IRQ_GERROR, error); + } + + trace_smmuv3_cmdq_consume_out(s->cmdq.wrap.prod, s->cmdq.prod, + s->cmdq.wrap.cons, s->cmdq.cons); + + return 0; +} + +/** + * GERROR is updated when raising an interrupt, GERRORN will be updated + * by SW and should match GERROR before normal operation resumes. + */ +static void smmu_irq_clear(SMMUV3State *s, uint64_t gerrorn) +{ + int irq = SMMU_IRQ_GERROR; + uint32_t toggled; + + toggled = smmu_read32_reg(s, SMMU_REG_GERRORN) ^ gerrorn; + + while (toggled) { + irq = ctz32(toggled); + + qemu_irq_lower(s->irq[irq]); + + toggled &= toggled - 1; + } +} + +static int smmu_evtq_update(SMMUV3State *s) +{ + if (!smmu_enabled(s)) { + return 0; + } + + if (!smmu_is_q_empty(s, &s->evtq)) { + if (smmu_evt_irq_enabled(s)) { + smmu_irq_raise(s, SMMU_IRQ_EVTQ, 0); + } + } + + if (smmu_is_q_empty(s, &s->evtq)) { + smmu_irq_clear(s, SMMU_GERROR_EVENTQ); + } + + return 1; +} + +static void smmu_create_event(SMMUV3State *s, hwaddr iova, + uint32_t sid, bool is_write, int error); + +static void smmu_update(SMMUV3State *s) +{ + int error = 0; + + /* SMMU starts processing commands even when not enabled */ + if (!smmu_enabled(s)) { + goto check_cmdq; + } + + /* EVENT Q updates takes more priority */ + if ((smmu_evt_q_enabled(s)) && !smmu_is_q_empty(s, &s->evtq)) { + trace_smmuv3_update(smmu_is_q_empty(s, &s->evtq), s->evtq.prod, + s->evtq.cons, s->evtq.wrap.prod, s->evtq.wrap.cons); + error = smmu_evtq_update(s); + } + + if (error) { + /* TODO: May be in future we create proper event queue entry */ + /* an error condition is not a recoverable event, like other devices */ + error_report("An unfavourable condition"); + smmu_create_event(s, 0, 0, 0, error); + } + +check_cmdq: + if (smmu_cmd_q_enabled(s) && !SMMU_CMDQ_ERR(s)) { + smmu_cmdq_consume(s); + } else { + trace_smmuv3_update_check_cmd(SMMU_CMDQ_ERR(s)); + } + +} + +static void smmu_update_irq(SMMUV3State *s, uint64_t addr, uint64_t val) +{ + smmu_irq_clear(s, val); + + smmu_write32_reg(s, SMMU_REG_GERRORN, val); + + trace_smmuv3_update_irq(smmu_is_irq_pending(s, 0), + smmu_read32_reg(s, SMMU_REG_GERROR), + smmu_read32_reg(s, SMMU_REG_GERRORN)); + + /* Clear only when no more left */ + if (!smmu_is_irq_pending(s, 0)) { + qemu_irq_lower(s->irq[0]); + } +} + +#define SMMU_ID_REG_INIT(s, reg, d) do { \ + s->regs[reg >> 2] = d; \ + } while (0) + +static void smmuv3_id_reg_init(SMMUV3State *s) +{ + uint32_t data = + SMMU_IDR0_STLEVEL << SMMU_IDR0_STLEVEL_SHIFT | + SMMU_IDR0_TERM << SMMU_IDR0_TERM_SHIFT | + SMMU_IDR0_STALL << SMMU_IDR0_STALL_SHIFT | + SMMU_IDR0_VMID16 << SMMU_IDR0_VMID16_SHIFT | + SMMU_IDR0_PRI << SMMU_IDR0_PRI_SHIFT | + SMMU_IDR0_ASID16 << SMMU_IDR0_ASID16_SHIFT | + SMMU_IDR0_ATS << SMMU_IDR0_ATS_SHIFT | + SMMU_IDR0_HYP << SMMU_IDR0_HYP_SHIFT | + SMMU_IDR0_HTTU << SMMU_IDR0_HTTU_SHIFT | + SMMU_IDR0_COHACC << SMMU_IDR0_COHACC_SHIFT | + SMMU_IDR0_TTF << SMMU_IDR0_TTF_SHIFT | + SMMU_IDR0_S1P << SMMU_IDR0_S1P_SHIFT | + SMMU_IDR0_S2P << SMMU_IDR0_S2P_SHIFT; + + SMMU_ID_REG_INIT(s, SMMU_REG_IDR0, data); + +#define SMMU_QUEUE_SIZE_LOG2 19 + data = + 1 << 27 | /* Attr Types override */ + SMMU_QUEUE_SIZE_LOG2 << 21 | /* Cmd Q size */ + SMMU_QUEUE_SIZE_LOG2 << 16 | /* Event Q size */ + SMMU_QUEUE_SIZE_LOG2 << 11 | /* PRI Q size */ + 0 << 6 | /* SSID not supported */ + SMMU_IDR1_SIDSIZE; + + SMMU_ID_REG_INIT(s, SMMU_REG_IDR1, data); + + data = + SMMU_IDR5_GRAN << SMMU_IDR5_GRAN_SHIFT | SMMU_IDR5_OAS; + + SMMU_ID_REG_INIT(s, SMMU_REG_IDR5, data); + +} + +static void smmuv3_init(SMMUV3State *s) +{ + smmuv3_id_reg_init(s); /* Update ID regs alone */ + + s->sid_size = SMMU_IDR1_SIDSIZE; + + s->cmdq.entries = (smmu_read32_reg(s, SMMU_REG_IDR1) >> 21) & 0x1f; + s->cmdq.ent_size = sizeof(Cmd); + s->evtq.entries = (smmu_read32_reg(s, SMMU_REG_IDR1) >> 16) & 0x1f; + s->evtq.ent_size = sizeof(Evt); +} + +/* + * All SMMU data structures are little endian, and are aligned to 8 bytes + * L1STE/STE/L1CD/CD, Queue entries in CMDQ/EVTQ/PRIQ + */ +static inline int smmu_get_ste(SMMUV3State *s, hwaddr addr, Ste *buf) +{ + int ret; + + trace_smmuv3_get_ste(addr); + ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf)); + dump_ste(buf); + return ret; +} + +/* + * For now we only support CD with a single entry, 'ssid' is used to identify + * otherwise + */ +static inline int smmu_get_cd(SMMUV3State *s, Ste *ste, uint32_t ssid, Cd *buf) +{ + hwaddr addr = STE_CTXPTR(ste); + int ret; + + if (STE_S1CDMAX(ste) != 0) { + error_report("Multilevel Ctx Descriptor not supported yet"); + } + + ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf)); + + trace_smmuv3_get_cd(addr); + dump_cd(cd); + + return ret; +} + +/** + * is_ste_consistent - Check validity of STE + * according to 6.2.1 Validty of STE + * TODO: check the relevance of each check and compliance + * with this spec chapter + */ +static int is_ste_consistent(SMMUV3State *s, Ste *ste) +{ + uint32_t _config = STE_CONFIG(ste); + uint32_t ste_vmid, ste_eats, ste_s2s, ste_s1fmt, ste_s2aa64, ste_s1cdmax; + uint32_t ste_strw; + bool strw_unused, addr_out_of_range, granule_supported; + bool config[] = {_config & 0x1, _config & 0x2, _config & 0x3}; + + ste_vmid = STE_S2VMID(ste); + ste_eats = STE_EATS(ste); /* Enable PCIe ATS trans */ + ste_s2s = STE_S2S(ste); + ste_s1fmt = STE_S1FMT(ste); + ste_s2aa64 = STE_S2AA64(ste); + ste_s1cdmax = STE_S1CDMAX(ste); /*CD bit # S1ContextPtr */ + ste_strw = STE_STRW(ste); /* stream world control */ + + if (!STE_VALID(ste)) { + error_report("STE NOT valid"); + return false; + } + + granule_supported = is_s2granule_valid(ste); + + /* As S1/S2 combinations are supported do not check + * corresponding STE config values */ + + if (!config[2]) { + /* Report abort to device, no event recorded */ + error_report("STE config 0b000 not implemented"); + return false; + } + + if (!SMMU_IDR1_SIDSIZE && ste_s1cdmax && config[0] && + !SMMU_IDR0_CD2L && (ste_s1fmt == 1 || ste_s1fmt == 2)) { + error_report("STE inconsistant, CD mismatch"); + return false; + } + if (SMMU_IDR0_ATS && ((_config & 0x3) == 0) && + ((ste_eats == 2 && (_config != 0x7 || ste_s2s)) || + (ste_eats == 1 && !ste_s2s))) { + error_report("STE inconsistant, EATS/S2S mismatch"); + return false; + } + if (config[0] && (SMMU_IDR1_SIDSIZE && + (ste_s1cdmax > SMMU_IDR1_SIDSIZE))) { + error_report("STE inconsistant, SSID out of range"); + return false; + } + + strw_unused = (!SMMU_IDR0_S1P || !SMMU_IDR0_HYP || (_config == 4)); + + addr_out_of_range = STE_S2TTB(ste) > MAX_PA(ste); + + if (has_stage2(ste)) { + if ((ste_s2aa64 && !is_s2granule_valid(ste)) || + (!ste_s2aa64 && !(SMMU_IDR0_TTF & 0x1)) || + (ste_s2aa64 && !(SMMU_IDR0_TTF & 0x2)) || + ((STE_S2HA(ste) || STE_S2HD(ste)) && !ste_s2aa64) || + ((STE_S2HA(ste) || STE_S2HD(ste)) && !SMMU_IDR0_HTTU) || + (STE_S2HD(ste) && (SMMU_IDR0_HTTU == 1)) || addr_out_of_range) { + error_report("STE inconsistant"); + trace_smmuv3_is_ste_consistent(config[1], granule_supported, + addr_out_of_range, ste_s2aa64, + STE_S2HA(ste), STE_S2HD(ste), + STE_S2TTB(ste)); + return false; + } + } + if (SMMU_IDR0_S2P && (config[0] == 0 && config[1]) && + (strw_unused || !ste_strw) && !SMMU_IDR0_VMID16 && !(ste_vmid >> 8)) { + error_report("STE inconsistant, VMID out of range"); + return false; + } + + return true; +} + +/** + * smmu_find_ste - Return the stream table entry associated + * to the sid + * + * @s: smmuv3 handle + * @sid: stream ID + * @ste: returned stream table entry + * Supports linear and 2-level stream table + */ +static int smmu_find_ste(SMMUV3State *s, uint16_t sid, Ste *ste) +{ + hwaddr addr; + + trace_smmuv3_find_ste(sid, s->features, s->sid_split); + /* Check SID range */ + if (sid > (1 << s->sid_size)) { + return SMMU_EVT_C_BAD_SID; + } + if (s->features & SMMU_FEATURE_2LVL_STE) { + int l1_ste_offset, l2_ste_offset, max_l2_ste, span; + hwaddr l1ptr, l2ptr; + STEDesc l1std; + + l1_ste_offset = sid >> s->sid_split; + l2_ste_offset = sid & ((1 << s->sid_split) - 1); + l1ptr = (hwaddr)(s->strtab_base + l1_ste_offset * sizeof(l1std)); + smmu_read_sysmem(l1ptr, &l1std, sizeof(l1std), false); + span = L1STD_SPAN(&l1std); + + if (!span) { + /* l2ptr is not valid */ + error_report("invalid sid=%d (L1STD span=0)", sid); + return SMMU_EVT_C_BAD_SID; + } + max_l2_ste = (1 << span) - 1; + l2ptr = L1STD_L2PTR(&l1std); + trace_smmuv3_find_ste_2lvl(s->strtab_base, l1ptr, l1_ste_offset, + l2ptr, l2_ste_offset, max_l2_ste); + if (l2_ste_offset > max_l2_ste) { + error_report("l2_ste_offset=%d > max_l2_ste=%d", + l2_ste_offset, max_l2_ste); + return SMMU_EVT_C_BAD_STE; + } + addr = L1STD_L2PTR(&l1std) + l2_ste_offset * sizeof(*ste); + } else { + addr = s->strtab_base + sid * sizeof(*ste); + } + + if (smmu_get_ste(s, addr, ste)) { + error_report("Unable to Fetch STE"); + return SMMU_EVT_F_UUT; + } + + return 0; +} + +/** + * smmu_cfg_populate_s1 - Populate the stage 1 translation config + * from the context descriptor + */ +static int smmu_cfg_populate_s1(SMMUTransCfg *cfg, Cd *cd) +{ + bool s1a64 = CD_AARCH64(cd); + int epd0 = CD_EPD0(cd); + int tg; + + cfg->stage = 1; + tg = epd0 ? CD_TG1(cd) : CD_TG0(cd); + cfg->tsz = epd0 ? CD_T1SZ(cd) : CD_T0SZ(cd); + cfg->ttbr = epd0 ? CD_TTB1(cd) : CD_TTB0(cd); + cfg->oas = oas2bits(CD_IPS(cd)); + + if (s1a64) { + cfg->tsz = MIN(cfg->tsz, 39); + cfg->tsz = MAX(cfg->tsz, 16); + } + cfg->granule_sz = tg2granule(tg, epd0); + + cfg->oas = MIN(oas2bits(SMMU_IDR5_OAS), cfg->oas); + /* fix ttbr - make top bits zero*/ + cfg->ttbr = extract64(cfg->ttbr, 0, cfg->oas); + cfg->aa64 = s1a64; + + trace_smmuv3_cfg_stage(cfg->stage, cfg->oas, cfg->tsz, cfg->ttbr, + cfg->aa64, cfg->granule_sz); + + return 0; +} + +/** + * smmu_cfg_populate_s2 - Populate the stage 2 translation config + * from the Stream Table Entry + */ +static int smmu_cfg_populate_s2(SMMUTransCfg *cfg, Ste *ste) +{ + bool s2a64 = STE_S2AA64(ste); + int tg; + + if (cfg->stage) { + error_report("%s nested S1 + S2 is not supported", __func__); + } else { + /* S2 only */ + cfg->stage = 2; + } + + tg = STE_S2TG(ste); + cfg->tsz = STE_S2T0SZ(ste); + cfg->ttbr = STE_S2TTB(ste); + cfg->oas = pa_range(ste); + + cfg->aa64 = s2a64; + + if (s2a64) { + cfg->tsz = MIN(cfg->tsz, 39); + cfg->tsz = MAX(cfg->tsz, 16); + } + cfg->granule_sz = tg2granule(tg, 0); + + cfg->oas = MIN(oas2bits(SMMU_IDR5_OAS), cfg->oas); + /* fix ttbr - make top bits zero*/ + cfg->ttbr = extract64(cfg->ttbr, 0, cfg->oas); + + trace_smmuv3_cfg_stage(cfg->stage, cfg->oas, cfg->tsz, cfg->ttbr, + cfg->aa64, cfg->granule_sz); + + return 0; +} + +/** + * smmu_cfg_populate - Populates the translation config corresponding + * to the STE and CD content + */ +static int smmu_cfg_populate(Ste *ste, Cd *cd, SMMUTransCfg *cfg) +{ + int ret = 0; + + if (is_ste_bypass(ste)) { + return ret; + } + + if (has_stage1(ste)) { + ret = smmu_cfg_populate_s1(cfg, cd); + if (ret) { + return ret; + } + } + if (has_stage2(ste)) { + ret = smmu_cfg_populate_s2(cfg, ste); + if (ret) { + return ret; + } + } + return ret; +} + +static SMMUEvtErr smmu_walk_pgtable(SMMUV3State *s, SMMUTransCfg *cfg, + IOMMUTLBEntry *tlbe, bool is_write) +{ + SMMUState *sys = SMMU_SYS_DEV(s); + SMMUBaseClass *sbc = SMMU_DEVICE_GET_CLASS(sys); + SMMUEvtErr error = 0; + uint32_t page_size = 0, perm = 0; + + trace_smmuv3_walk_pgtable(tlbe->iova, is_write); + + if (!cfg->stage) { + return 0; + } + + cfg->input = tlbe->iova; + + if (cfg->aa64) { + error = sbc->translate_64(cfg, &page_size, &perm, is_write); + } else { + error = sbc->translate_32(cfg, &page_size, &perm, is_write); + } + + if (error) { + error_report("PTW failed for iova=0x%"PRIx64" is_write=%d (%d)", + cfg->input, is_write, error); + goto exit; + } + tlbe->translated_addr = cfg->output; + tlbe->addr_mask = page_size - 1; + tlbe->perm = perm; + + trace_smmuv3_walk_pgtable_out(tlbe->translated_addr, + tlbe->addr_mask, tlbe->perm); +exit: + return error; +} + +static MemTxResult smmu_write_evtq(SMMUV3State *s, Evt *evt) +{ + SMMUQueue *q = &s->evtq; + int ret = smmu_q_write(s, q, evt); + uint32_t val = 0; + + val |= (q->wrap.prod << q->shift) | q->prod; + + smmu_write32_reg(s, SMMU_REG_EVTQ_PROD, val); + + return ret; +} + +/* + * Events created on the EventQ + */ +static void smmu_create_event(SMMUV3State *s, hwaddr iova, + uint32_t sid, bool is_write, int error) +{ + SMMUQueue *q = &s->evtq; + uint64_t head; + Evt evt; + + if (!smmu_evt_q_enabled(s)) { + return; + } + + EVT_SET_TYPE(&evt, error); + EVT_SET_SID(&evt, sid); + + switch (error) { + case SMMU_EVT_F_UUT: + case SMMU_EVT_C_BAD_STE: + break; + case SMMU_EVT_C_BAD_CD: + case SMMU_EVT_F_CD_FETCH: + break; + case SMMU_EVT_F_TRANS_FORBIDDEN: + case SMMU_EVT_F_WALK_EXT_ABRT: + EVT_SET_INPUT_ADDR(&evt, iova); + default: + break; + } + + smmu_write_evtq(s, &evt); + + head = Q_IDX(q, q->prod); + + if (smmu_is_q_full(s, &s->evtq)) { + head = q->prod ^ (1 << 31); /* Set overflow */ + } + + smmu_write32_reg(s, SMMU_REG_EVTQ_PROD, head); + + smmu_irq_raise(s, SMMU_IRQ_EVTQ, 0); +} + +/* + * TR - Translation Request + * TT - Translated Tansaction + * OT - Other Transaction + */ +static IOMMUTLBEntry +smmuv3_translate(MemoryRegion *mr, hwaddr addr, bool is_write) +{ + SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu); + SMMUV3State *s = sdev->smmu; + SMMUTransCfg transcfg = {}; + uint16_t sid = 0; + Ste ste; + Cd cd; + SMMUEvtErr error = 0; + + IOMMUTLBEntry entry = { + .target_as = &address_space_memory, + .iova = addr, + .translated_addr = addr, + .addr_mask = ~(hwaddr)0, + .perm = IOMMU_NONE, + }; + + sid = smmu_get_sid(sdev); + + /* SMMU Bypass, We allow traffic through if SMMU is disabled */ + if (!smmu_enabled(s)) { + trace_smmuv3_translate_bypass(mr->name, sid, addr, is_write); + goto out; + } + + trace_smmuv3_translate_in(sid, pci_bus_num(sdev->bus), s->strtab_base); + + /* Fetch & Check STE */ + error = smmu_find_ste(s, sid, &ste); + if (error) { + goto error_out; /* F_STE_FETCH or F_CFG_CONFLICT */ + } + + if (STE_VALID(&ste) && is_ste_bypass(&ste)) { + trace_smmuv3_translate_bypass(mr->name, sid, addr, is_write); + goto out; + } + + if (!is_ste_consistent(s, &ste)) { + error = SMMU_EVT_C_BAD_STE; + goto error_out; + } + + if (has_stage1(&ste)) { /* Stage 1 */ + smmu_get_cd(s, &ste, 0, &cd); /* We dont have SSID yet, so 0 */ + + if (!is_cd_valid(s, &ste, &cd)) { + error = SMMU_EVT_C_BAD_CD; + goto error_out; + } + } + + smmu_cfg_populate(&ste, &cd, &transcfg); + + /* Walk Stage1, if S2 is enabled, S2 walked for Every access on S1 */ + error = smmu_walk_pgtable(s, &transcfg, &entry, is_write); + + entry.perm = is_write ? IOMMU_RW : IOMMU_RO; + + trace_smmuv3_translate_ok(mr->name, sid, addr, + entry.translated_addr, entry.perm); + +error_out: + if (error > 1) { + error_report("Translation Error: %x", error); + smmu_create_event(s, entry.iova, sid, is_write, error); + } + +out: + return entry; +} + +static inline void smmu_update_base_reg(SMMUV3State *s, uint64_t *base, + uint64_t val) +{ + *base = val & ~(SMMU_BASE_RA | 0x3fULL); +} + +static void smmu_update_qreg(SMMUV3State *s, SMMUQueue *q, hwaddr reg, + uint32_t off, uint64_t val, unsigned size) +{ + if (size == 8 && off == 0) { + smmu_write64_reg(s, reg, val); + } else { + smmu_write_reg(s, reg, val); + } + + switch (off) { + case 0: /* BASE register */ + val = smmu_read64_reg(s, reg); + q->shift = val & 0x1f; + q->entries = 1 << (q->shift); + smmu_update_base_reg(s, &q->base, val); + break; + + case 4: /* CONS */ + q->cons = Q_IDX(q, val); + q->wrap.cons = val >> q->shift; + trace_smmuv3_update_qreg(q->cons, val); + break; + + case 8: /* PROD */ + q->prod = Q_IDX(q, val); + q->wrap.prod = val >> q->shift; + break; + } + + switch (reg) { + case SMMU_REG_CMDQ_PROD: /* should be only for CMDQ_PROD */ + case SMMU_REG_CMDQ_CONS: /* but we do it anyway */ + smmu_update(s); + break; + } +} + +static void smmu_write_mmio_fixup(SMMUV3State *s, hwaddr *addr) +{ + switch (*addr) { + case 0x100a8: case 0x100ac: /* Aliasing => page0 registers */ + case 0x100c8: case 0x100cc: + *addr ^= (hwaddr)0x10000; + } +} + +static void smmu_write_mmio(void *opaque, hwaddr addr, + uint64_t val, unsigned size) +{ + SMMUState *sys = opaque; + SMMUV3State *s = SMMU_V3_DEV(sys); + bool update = false; + + smmu_write_mmio_fixup(s, &addr); + + trace_smmuv3_write_mmio(addr, val); + + switch (addr) { + case 0xFDC ... 0xFFC: + case SMMU_REG_IDR0 ... SMMU_REG_IDR5: + trace_smmuv3_write_mmio_idr(addr, val); + return; + + case SMMU_REG_GERRORN: + smmu_update_irq(s, addr, val); + return; + + case SMMU_REG_CR0: + smmu_write32_reg(s, SMMU_REG_CR0, val); + smmu_write32_reg(s, SMMU_REG_CR0_ACK, val); + update = true; + break; + + case SMMU_REG_IRQ_CTRL: + smmu_write32_reg(s, SMMU_REG_IRQ_CTRL_ACK, val); + update = true; + break; + + case SMMU_REG_STRTAB_BASE: + smmu_update_base_reg(s, &s->strtab_base, val); + return; + + case SMMU_REG_STRTAB_BASE_CFG: + if (((val >> 16) & 0x3) == 0x1) { + s->sid_split = (val >> 6) & 0x1f; + s->features |= SMMU_FEATURE_2LVL_STE; + } + break; + + case SMMU_REG_CMDQ_PROD: + case SMMU_REG_CMDQ_CONS: + case SMMU_REG_CMDQ_BASE: + case SMMU_REG_CMDQ_BASE + 4: + smmu_update_qreg(s, &s->cmdq, addr, addr - SMMU_REG_CMDQ_BASE, + val, size); + return; + + case SMMU_REG_EVTQ_CONS: /* fallthrough */ + { + SMMUQueue *evtq = &s->evtq; + evtq->cons = Q_IDX(evtq, val); + evtq->wrap.cons = Q_WRAP(evtq, val); + + trace_smmuv3_write_mmio_evtq_cons_bef_clear(evtq->prod, evtq->cons, + evtq->wrap.prod, + evtq->wrap.cons); + if (smmu_is_q_empty(s, &s->evtq)) { + trace_smmuv3_write_mmio_evtq_cons_after_clear(evtq->prod, + evtq->cons, + evtq->wrap.prod, + evtq->wrap.cons); + qemu_irq_lower(s->irq[SMMU_IRQ_EVTQ]); + } + } + case SMMU_REG_EVTQ_BASE: + case SMMU_REG_EVTQ_BASE + 4: + case SMMU_REG_EVTQ_PROD: + smmu_update_qreg(s, &s->evtq, addr, addr - SMMU_REG_EVTQ_BASE, + val, size); + return; + + case SMMU_REG_PRIQ_CONS: + case SMMU_REG_PRIQ_BASE: + case SMMU_REG_PRIQ_BASE + 4: + case SMMU_REG_PRIQ_PROD: + smmu_update_qreg(s, &s->priq, addr, addr - SMMU_REG_PRIQ_BASE, + val, size); + return; + } + + if (size == 8) { + smmu_write_reg(s, addr, val); + } else { + smmu_write32_reg(s, addr, (uint32_t)val); + } + + if (update) { + smmu_update(s); + } +} + +static uint64_t smmu_read_mmio(void *opaque, hwaddr addr, unsigned size) +{ + SMMUState *sys = opaque; + SMMUV3State *s = SMMU_V3_DEV(sys); + uint64_t val; + + smmu_write_mmio_fixup(s, &addr); + + /* Primecell/Corelink ID registers */ + switch (addr) { + case 0xFF0 ... 0xFFC: + case 0xFDC ... 0xFE4: + val = 0; + error_report("addr:0x%"PRIx64" val:0x%"PRIx64, addr, val); + break; + + default: + val = (uint64_t)smmu_read32_reg(s, addr); + break; + + case SMMU_REG_STRTAB_BASE ... SMMU_REG_CMDQ_BASE: + case SMMU_REG_EVTQ_BASE: + case SMMU_REG_PRIQ_BASE ... SMMU_REG_PRIQ_IRQ_CFG1: + val = smmu_read64_reg(s, addr); + break; + } + + trace_smmuv3_read_mmio(addr, val, s->cmdq.cons); + return val; +} + +static const MemoryRegionOps smmu_mem_ops = { + .read = smmu_read_mmio, + .write = smmu_write_mmio, + .endianness = DEVICE_LITTLE_ENDIAN, + .valid = { + .min_access_size = 4, + .max_access_size = 8, + }, +}; + +static void smmu_init_irq(SMMUV3State *s, SysBusDevice *dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(s->irq); i++) { + sysbus_init_irq(dev, &s->irq[i]); + } +} + +static AddressSpace *smmu_find_add_as(PCIBus *bus, void *opaque, int devfn) +{ + SMMUState *s = opaque; + uintptr_t key = (uintptr_t)bus; + SMMUPciBus *sbus = g_hash_table_lookup(s->smmu_as_by_busptr, &key); + SMMUDevice *sdev; + + if (!sbus) { + uintptr_t *new_key = g_malloc(sizeof(*new_key)); + + *new_key = (uintptr_t)bus; + sbus = g_malloc0(sizeof(SMMUPciBus) + + sizeof(SMMUDevice *) * SMMU_PCI_DEVFN_MAX); + sbus->bus = bus; + g_hash_table_insert(s->smmu_as_by_busptr, new_key, sbus); + } + + sdev = sbus->pbdev[devfn]; + if (!sdev) { + sdev = sbus->pbdev[devfn] = g_malloc0(sizeof(SMMUDevice)); + + sdev->smmu = s; + sdev->bus = bus; + sdev->devfn = devfn; + + memory_region_init_iommu(&sdev->iommu, OBJECT(s), + &s->iommu_ops, TYPE_SMMU_V3_DEV, + UINT64_MAX); + address_space_init(&sdev->as, &sdev->iommu, TYPE_SMMU_V3_DEV); + } + + return &sdev->as; + +} + +static void smmu_init_iommu_as(SMMUV3State *sys) +{ + SMMUState *s = SMMU_SYS_DEV(sys); + PCIBus *pcibus = pci_find_primary_bus(); + + if (pcibus) { + pci_setup_iommu(pcibus, smmu_find_add_as, s); + } else { + error_report("No PCI bus, SMMU is not registered"); + } +} + +static void smmu_reset(DeviceState *dev) +{ + SMMUV3State *s = SMMU_V3_DEV(dev); + smmuv3_init(s); +} + +static int smmu_populate_internal_state(void *opaque, int version_id) +{ + SMMUV3State *s = opaque; + + smmu_update(s); + return 0; +} + +static void smmu_realize(DeviceState *d, Error **errp) +{ + SMMUState *sys = SMMU_SYS_DEV(d); + SMMUV3State *s = SMMU_V3_DEV(sys); + SysBusDevice *dev = SYS_BUS_DEVICE(d); + + sys->iommu_ops.translate = smmuv3_translate; + sys->iommu_ops.notify_flag_changed = NULL; + /* Register Access */ + memset(sys->smmu_as_by_bus_num, 0, sizeof(sys->smmu_as_by_bus_num)); + memory_region_init_io(&sys->iomem, OBJECT(s), + &smmu_mem_ops, sys, TYPE_SMMU_V3_DEV, 0x20000); + + sys->smmu_as_by_busptr = g_hash_table_new_full(smmu_uint64_hash, + smmu_uint64_equal, + g_free, g_free); + sysbus_init_mmio(dev, &sys->iomem); + + smmu_init_irq(s, dev); + + smmu_init_iommu_as(s); +} + +static const VMStateDescription vmstate_smmuv3 = { + .name = "smmuv3", + .version_id = 1, + .minimum_version_id = 1, + .post_load = smmu_populate_internal_state, + .fields = (VMStateField[]) { + VMSTATE_UINT64_ARRAY(regs, SMMUV3State, SMMU_NREGS), + VMSTATE_END_OF_LIST(), + }, +}; + +static void smmuv3_instance_init(Object *obj) +{ + /* Nothing much to do here as of now */ +} + +static void smmuv3_class_init(ObjectClass *klass, void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + + dc->reset = smmu_reset; + dc->vmsd = &vmstate_smmuv3; + dc->realize = smmu_realize; +} + +static const TypeInfo smmuv3_type_info = { + .name = TYPE_SMMU_V3_DEV, + .parent = TYPE_SMMU_DEV_BASE, + .instance_size = sizeof(SMMUV3State), + .instance_init = smmuv3_instance_init, + .class_data = NULL, + .class_size = sizeof(SMMUV3Class), + .class_init = smmuv3_class_init, +}; + +static void smmuv3_register_types(void) +{ + type_register(&smmuv3_type_info); +} + +type_init(smmuv3_register_types) + diff --git a/hw/arm/trace-events b/hw/arm/trace-events index 1d53ad0..d3aeaef 100644 --- a/hw/arm/trace-events +++ b/hw/arm/trace-events @@ -14,3 +14,35 @@ smmu_page_walk_level_block_pte(uint64_t pte, uint64_t address) "pte=0x%"PRIx64" smmu_page_walk_level_table_pte(uint64_t pte, uint64_t address) "pte=0x%"PRIx64" next table address = 0x%"PRIx64 smmu_get_pte(uint64_t baseaddr, int index, uint64_t pteaddr, uint64_t pte) "baseaddr=0x%"PRIx64" index=0x%x, pteaddr=0x%"PRIx64", pte=0x%"PRIx64 smmu_set_translated_address(hwaddr iova, hwaddr va) "iova = 0x%"PRIx64" -> pa = 0x%"PRIx64 + +#hw/arm/smmuv3.c +smmuv3_irq_update(uint32_t error, uint32_t gerror, uint32_t gerrorn) "<<<< error:0x%x gerror:0x%x gerrorn:0x%x" +smmuv3_irq_raise(int irq) "irq:%d" +smmuv3_unhandled_cmd(uint32_t type) "Unhandled command type=%d" +smmuv3_cmdq_consume(int error) "CMDQ_ERR: %d" +smmuv3_cmdq_consume_details(hwaddr base, uint32_t cons, uint32_t prod, uint32_t word, uint8_t wrap_cons) "CMDQ base: 0x%"PRIx64" cons:%d prod:%d val:0x%x wrap:%d" +smmuv3_cmdq_consume_sev(void) "CMD_SYNC CS=SEV not supported, ignoring" +smmuv3_cmdq_consume_out(uint8_t prod_wrap, uint32_t prod, uint8_t cons_wrap, uint32_t cons) "prod_wrap:%d, prod:0x%x cons_wrap:%d cons:0x%x" +smmuv3_update(bool is_empty, uint32_t prod, uint32_t cons, uint8_t prod_wrap, uint8_t cons_wrap) "q empty:%d prod:%d cons:%d p.wrap:%d p.cons:%d" +smmuv3_update_check_cmd(int error) "cmdq not enabled or error :0x%x" +smmuv3_update_irq(bool is_pending, uint32_t gerror, uint32_t gerrorn) "irq pend: %d gerror:0x%x gerrorn:0x%x" +smmuv3_is_ste_consistent(bool cfg, bool granule_supported, bool addr_oor, uint32_t aa64, int s2ha, int s2hd, uint64_t s2ttb ) "config[1]:%d gran:%d addr:%d aa64:%d s2ha:%d s2hd:%d s2ttb:0x%"PRIx64 +smmuv3_find_ste(uint16_t sid, uint32_t features, uint16_t sid_split) "SID:0x%x features:0x%x, sid_split:0x%x" +smmuv3_find_ste_2lvl(uint64_t strtab_base, hwaddr l1ptr, int l1_ste_offset, hwaddr l2ptr, int l2_ste_offset, int max_l2_ste) "strtab_base:%lx l1ptr:0x%"PRIx64" l1_off:0x%x, l2ptr:0x%"PRIx64" l2_off:0x%x max_l2_ste:%d" +smmuv3_get_ste(hwaddr addr) "STE addr: 0x%"PRIx64 +smmuv3_walk_pgtable(hwaddr iova, bool is_write) "Input addr: 0x%"PRIx64", is_write=%d" +smmuv3_walk_pgtable_out(hwaddr addr, uint32_t mask, int perm) "DONE: o/p addr:0x%"PRIx64" mask:0x%x perm:%d" +smmuv3_translate_bypass(const char *n, uint16_t sid, hwaddr addr, bool is_write) "%s sid=%d bypass iova:0x%"PRIx64" is_write=%d" +smmuv3_translate_in(uint16_t sid, int pci_bus_num, hwaddr strtab_base) "SID:0x%x bus:%d strtab_base:0x%"PRIx64 +smmuv3_get_cd(hwaddr addr) "CD addr: 0x%"PRIx64 +smmuv3_translate_ok(const char *n, uint16_t sid, hwaddr iova, hwaddr translated, int perm) "%s sid=%d iova=0x%"PRIx64" translated=0x%"PRIx64" perm=0x%x" +smmuv3_update_qreg(uint32_t cons, uint64_t val) "cons written : %d val:0x%"PRIx64 +smmuv3_write_mmio(hwaddr addr, uint64_t val) "addr: 0x%"PRIx64" val:0x%"PRIx64 +smmuv3_write_mmio_idr(hwaddr addr, uint64_t val) "write to RO/Unimpl reg %lx val64:%lx" +smmuv3_write_mmio_evtq_cons_bef_clear(uint32_t prod, uint32_t cons, uint8_t prod_wrap, uint8_t cons_wrap) "Before clearing interrupt prod:0x%x cons:0x%x prod.w:%d cons.w:%d" +smmuv3_write_mmio_evtq_cons_after_clear(uint32_t prod, uint32_t cons, uint8_t prod_wrap, uint8_t cons_wrap) "after clearing interrupt prod:0x%x cons:0x%x prod.w:%d cons.w:%d" +smmuv3_read_mmio(hwaddr addr, uint64_t val, uint32_t cons) "addr: 0x%"PRIx64" val:0x%"PRIx64" cmdq cons:%d" +smmuv3_dump_ste(int i, uint32_t word0, int j, uint32_t word1) "STE[%2d]: %#010x\t STE[%2d]: %#010x" +smmuv3_dump_cd(int i, uint32_t word0, int j, uint32_t word1) "CD[%2d]: %#010x\t CD[%2d]: %#010x" +smmuv3_dump_cmd(int i, uint32_t word0, int j, uint32_t word1) "CMD[%2d]: %#010x\t CMD[%2d]: %#010x" +smmuv3_cfg_stage(int s, uint32_t oas, uint32_t tsz, uint64_t ttbr, bool aa64, uint32_t granule_sz) "TransCFG stage:%d oas:%d tsz:%d ttbr:0x%"PRIx64" aa64:%d granule_sz:%d" diff --git a/include/hw/arm/smmuv3.h b/include/hw/arm/smmuv3.h new file mode 100644 index 0000000..1f7d78e --- /dev/null +++ b/include/hw/arm/smmuv3.h @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2014-2016 Broadcom Corporation + * Copyright (c) 2017 Red Hat, Inc. + * Written by Prem Mallappa, Eric Auger + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef HW_ARM_SMMUV3_H +#define HW_ARM_SMMUV3_H + +#include "hw/arm/smmu-common.h" + +#define SMMU_NREGS 0x200 + +typedef struct SMMUQueue { + hwaddr base; + uint32_t prod; + uint32_t cons; + union { + struct { + uint8_t prod:1; + uint8_t cons:1; + }; + uint8_t unused; + } wrap; + + uint16_t entries; /* Number of entries */ + uint8_t ent_size; /* Size of entry in bytes */ + uint8_t shift; /* Size in log2 */ +} SMMUQueue; + +typedef struct SMMUV3State { + SMMUState smmu_state; + +#define SMMU_FEATURE_2LVL_STE (1 << 0) + /* Local cache of most-frequently used register */ + uint32_t features; + uint16_t sid_size; + uint16_t sid_split; + uint64_t strtab_base; + + uint64_t regs[SMMU_NREGS]; + + qemu_irq irq[4]; + + SMMUQueue cmdq, evtq, priq; + + /* IOMMU Address space */ + MemoryRegion iommu; + AddressSpace iommu_as; + /* + * Bus number is not populated in the beginning, hence we need + * a mechanism to retrieve the corresponding address space for each + * pci device. + */ + GHashTable *smmu_as_by_busptr; +} SMMUV3State; + +typedef enum { + SMMU_IRQ_GERROR, + SMMU_IRQ_PRIQ, + SMMU_IRQ_EVTQ, + SMMU_IRQ_CMD_SYNC, +} SMMUIrq; + +typedef struct { + SMMUBaseClass smmu_base_class; +} SMMUV3Class; + +#define TYPE_SMMU_V3_DEV "smmuv3" +#define SMMU_V3_DEV(obj) OBJECT_CHECK(SMMUV3State, (obj), TYPE_SMMU_V3_DEV) +#define SMMU_V3_DEVICE_GET_CLASS(obj) \ + OBJECT_GET_CLASS(SMMUBaseClass, (obj), TYPE_SMMU_V3_DEV) + +#endif -- 2.5.5
