Author: br Date: Mon Nov 16 21:55:52 2020 New Revision: 367736 URL: https://svnweb.freebsd.org/changeset/base/367736
Log: Introduce IOMMU support for arm64 platform. This adds an arm64 iommu interface and a driver for Arm System Memory Management Unit version 3.2 (ARM SMMU v3.2) specified in ARM IHI 0070C document. Hardware overview is provided in the header of smmu.c file. The support is disabled by default. To enable add 'options IOMMU' to your kernel configuration file. The support was developed on Arm Neoverse N1 System Development Platform (ARM N1SDP), kindly provided by ARM Ltd. Currently, PCI-based devices and ACPI platforms are supported only. The support was tested on IOMMU-enabled Marvell SATA controller, Realtek Ethernet controller and a TI xHCI USB controller with a low to medium load only. Many thanks to Konstantin Belousov for help forming the generic IOMMU framework that is vital for this project; to Andrew Turner for adding IOMMU support to MSI interrupt code; to Mark Johnston for help with SMMU page management; to John Baldwin for explaining various IOMMU bits. Reviewed by: mmel Relnotes: yes Sponsored by: DARPA / AFRL Sponsored by: Innovate UK (Digital Security by Design programme) Differential Revision: https://reviews.freebsd.org/D24618 Added: head/sys/arm64/include/iommu.h (contents, props changed) head/sys/arm64/iommu/ head/sys/arm64/iommu/iommu.c (contents, props changed) head/sys/arm64/iommu/iommu.h (contents, props changed) head/sys/arm64/iommu/iommu_if.m (contents, props changed) head/sys/arm64/iommu/smmu.c (contents, props changed) head/sys/arm64/iommu/smmu_acpi.c (contents, props changed) head/sys/arm64/iommu/smmu_quirks.c (contents, props changed) head/sys/arm64/iommu/smmureg.h (contents, props changed) head/sys/arm64/iommu/smmuvar.h (contents, props changed) Modified: head/sys/conf/files.arm64 Added: head/sys/arm64/include/iommu.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/arm64/include/iommu.h Mon Nov 16 21:55:52 2020 (r367736) @@ -0,0 +1,11 @@ +/*- + * This file is in the public domain. + */ +/* $FreeBSD$ */ + +#ifndef _MACHINE_IOMMU_H_ +#define _MACHINE_IOMMU_H_ + +#include <arm64/iommu/iommu.h> + +#endif /* !_MACHINE_IOMMU_H_ */ Added: head/sys/arm64/iommu/iommu.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/arm64/iommu/iommu.c Mon Nov 16 21:55:52 2020 (r367736) @@ -0,0 +1,397 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2020 Ruslan Bukin <b...@bsdpad.com> + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory (Department of Computer Science and + * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the + * DARPA SSITH research programme. + * + * Portions of this work was supported by Innovate UK project 105694, + * "Digital Security by Design (DSbD) Technology Platform Prototype". + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "opt_platform.h" + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/bus.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/memdesc.h> +#include <sys/tree.h> +#include <sys/taskqueue.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/sysctl.h> +#include <vm/vm.h> + +#include <dev/pci/pcireg.h> +#include <dev/pci/pcivar.h> +#include <machine/bus.h> +#include <dev/iommu/busdma_iommu.h> +#include <machine/vmparam.h> + +#include "iommu.h" +#include "iommu_if.h" + +static MALLOC_DEFINE(M_IOMMU, "IOMMU", "IOMMU framework"); + +#define IOMMU_LIST_LOCK() mtx_lock(&iommu_mtx) +#define IOMMU_LIST_UNLOCK() mtx_unlock(&iommu_mtx) +#define IOMMU_LIST_ASSERT_LOCKED() mtx_assert(&iommu_mtx, MA_OWNED) + +#define dprintf(fmt, ...) + +static struct mtx iommu_mtx; + +struct iommu_entry { + struct iommu_unit *iommu; + LIST_ENTRY(iommu_entry) next; +}; +static LIST_HEAD(, iommu_entry) iommu_list = LIST_HEAD_INITIALIZER(iommu_list); + +static int +iommu_domain_unmap_buf(struct iommu_domain *iodom, iommu_gaddr_t base, + iommu_gaddr_t size, int flags) +{ + struct iommu_unit *iommu; + int error; + + iommu = iodom->iommu; + + error = IOMMU_UNMAP(iommu->dev, iodom, base, size); + + return (error); +} + +static int +iommu_domain_map_buf(struct iommu_domain *iodom, iommu_gaddr_t base, + iommu_gaddr_t size, vm_page_t *ma, uint64_t eflags, int flags) +{ + struct iommu_unit *iommu; + vm_prot_t prot; + vm_offset_t va; + int error; + + dprintf("%s: base %lx, size %lx\n", __func__, base, size); + + prot = 0; + if (eflags & IOMMU_MAP_ENTRY_READ) + prot |= VM_PROT_READ; + if (eflags & IOMMU_MAP_ENTRY_WRITE) + prot |= VM_PROT_WRITE; + + va = base; + + iommu = iodom->iommu; + + error = IOMMU_MAP(iommu->dev, iodom, va, ma, size, prot); + + return (0); +} + +static const struct iommu_domain_map_ops domain_map_ops = { + .map = iommu_domain_map_buf, + .unmap = iommu_domain_unmap_buf, +}; + +static struct iommu_domain * +iommu_domain_alloc(struct iommu_unit *iommu) +{ + struct iommu_domain *iodom; + + iodom = IOMMU_DOMAIN_ALLOC(iommu->dev, iommu); + if (iodom == NULL) + return (NULL); + + iommu_domain_init(iommu, iodom, &domain_map_ops); + iodom->end = VM_MAXUSER_ADDRESS; + iodom->iommu = iommu; + iommu_gas_init_domain(iodom); + + return (iodom); +} + +static int +iommu_domain_free(struct iommu_domain *iodom) +{ + struct iommu_unit *iommu; + + iommu = iodom->iommu; + + IOMMU_LOCK(iommu); + + if ((iodom->flags & IOMMU_DOMAIN_GAS_INITED) != 0) { + IOMMU_DOMAIN_LOCK(iodom); + iommu_gas_fini_domain(iodom); + IOMMU_DOMAIN_UNLOCK(iodom); + } + + iommu_domain_fini(iodom); + + IOMMU_DOMAIN_FREE(iommu->dev, iodom); + IOMMU_UNLOCK(iommu); + + return (0); +} + +static void +iommu_tag_init(struct bus_dma_tag_iommu *t) +{ + bus_addr_t maxaddr; + + maxaddr = BUS_SPACE_MAXADDR; + + t->common.ref_count = 0; + t->common.impl = &bus_dma_iommu_impl; + t->common.alignment = 1; + t->common.boundary = 0; + t->common.lowaddr = maxaddr; + t->common.highaddr = maxaddr; + t->common.maxsize = maxaddr; + t->common.nsegments = BUS_SPACE_UNRESTRICTED; + t->common.maxsegsz = maxaddr; +} + +static struct iommu_ctx * +iommu_ctx_alloc(device_t dev, struct iommu_domain *iodom, bool disabled) +{ + struct iommu_unit *iommu; + struct iommu_ctx *ioctx; + + iommu = iodom->iommu; + + ioctx = IOMMU_CTX_ALLOC(iommu->dev, iodom, dev, disabled); + if (ioctx == NULL) + return (NULL); + + /* + * iommu can also be used for non-PCI based devices. + * This should be reimplemented as new newbus method with + * pci_get_rid() as a default for PCI device class. + */ + ioctx->rid = pci_get_rid(dev); + + return (ioctx); +} + +struct iommu_ctx * +iommu_get_ctx(struct iommu_unit *iommu, device_t requester, + uint16_t rid, bool disabled, bool rmrr) +{ + struct iommu_ctx *ioctx; + struct iommu_domain *iodom; + struct bus_dma_tag_iommu *tag; + + IOMMU_LOCK(iommu); + ioctx = IOMMU_CTX_LOOKUP(iommu->dev, requester); + if (ioctx) { + IOMMU_UNLOCK(iommu); + return (ioctx); + } + IOMMU_UNLOCK(iommu); + + /* + * In our current configuration we have a domain per each ctx. + * So allocate a domain first. + */ + iodom = iommu_domain_alloc(iommu); + if (iodom == NULL) + return (NULL); + + ioctx = iommu_ctx_alloc(requester, iodom, disabled); + if (ioctx == NULL) { + iommu_domain_free(iodom); + return (NULL); + } + + tag = ioctx->tag = malloc(sizeof(struct bus_dma_tag_iommu), + M_IOMMU, M_WAITOK | M_ZERO); + tag->owner = requester; + tag->ctx = ioctx; + tag->ctx->domain = iodom; + + iommu_tag_init(tag); + + ioctx->domain = iodom; + + return (ioctx); +} + +void +iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *ioctx) +{ + struct bus_dma_tag_iommu *tag; + + IOMMU_ASSERT_LOCKED(iommu); + + tag = ioctx->tag; + + IOMMU_CTX_FREE(iommu->dev, ioctx); + + free(tag, M_IOMMU); +} + +void +iommu_free_ctx(struct iommu_ctx *ioctx) +{ + struct iommu_unit *iommu; + struct iommu_domain *iodom; + int error; + + iodom = ioctx->domain; + iommu = iodom->iommu; + + IOMMU_LOCK(iommu); + iommu_free_ctx_locked(iommu, ioctx); + IOMMU_UNLOCK(iommu); + + /* Since we have a domain per each ctx, remove the domain too. */ + error = iommu_domain_free(iodom); + if (error) + device_printf(iommu->dev, "Could not free a domain\n"); +} + +static void +iommu_domain_free_entry(struct iommu_map_entry *entry, bool free) +{ + struct iommu_domain *iodom; + + iodom = entry->domain; + + IOMMU_DOMAIN_LOCK(iodom); + iommu_gas_free_space(iodom, entry); + IOMMU_DOMAIN_UNLOCK(iodom); + + if (free) + iommu_gas_free_entry(iodom, entry); + else + entry->flags = 0; +} + +void +iommu_domain_unload(struct iommu_domain *iodom, + struct iommu_map_entries_tailq *entries, bool cansleep) +{ + struct iommu_map_entry *entry, *entry1; + int error; + + TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { + KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0, + ("not mapped entry %p %p", iodom, entry)); + error = iodom->ops->unmap(iodom, entry->start, entry->end - + entry->start, cansleep ? IOMMU_PGF_WAITOK : 0); + KASSERT(error == 0, ("unmap %p error %d", iodom, error)); + TAILQ_REMOVE(entries, entry, dmamap_link); + iommu_domain_free_entry(entry, true); + } + + if (TAILQ_EMPTY(entries)) + return; + + panic("entries map is not empty"); +} + +int +iommu_register(struct iommu_unit *iommu) +{ + struct iommu_entry *entry; + + mtx_init(&iommu->lock, "IOMMU", NULL, MTX_DEF); + + entry = malloc(sizeof(struct iommu_entry), M_IOMMU, M_WAITOK | M_ZERO); + entry->iommu = iommu; + + IOMMU_LIST_LOCK(); + LIST_INSERT_HEAD(&iommu_list, entry, next); + IOMMU_LIST_UNLOCK(); + + iommu_init_busdma(iommu); + + return (0); +} + +int +iommu_unregister(struct iommu_unit *iommu) +{ + struct iommu_entry *entry, *tmp; + + IOMMU_LIST_LOCK(); + LIST_FOREACH_SAFE(entry, &iommu_list, next, tmp) { + if (entry->iommu == iommu) { + LIST_REMOVE(entry, next); + free(entry, M_IOMMU); + } + } + IOMMU_LIST_UNLOCK(); + + iommu_fini_busdma(iommu); + + mtx_destroy(&iommu->lock); + + return (0); +} + +struct iommu_unit * +iommu_find(device_t dev, bool verbose) +{ + struct iommu_entry *entry; + struct iommu_unit *iommu; + int error; + + IOMMU_LIST_LOCK(); + LIST_FOREACH(entry, &iommu_list, next) { + iommu = entry->iommu; + error = IOMMU_FIND(iommu->dev, dev); + if (error == 0) { + IOMMU_LIST_UNLOCK(); + return (entry->iommu); + } + } + IOMMU_LIST_UNLOCK(); + + return (NULL); +} + +void +iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free) +{ + + dprintf("%s\n", __func__); + + iommu_domain_free_entry(entry, free); +} + +static void +iommu_init(void) +{ + + mtx_init(&iommu_mtx, "IOMMU", NULL, MTX_DEF); +} + +SYSINIT(iommu, SI_SUB_DRIVERS, SI_ORDER_FIRST, iommu_init, NULL); Added: head/sys/arm64/iommu/iommu.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/arm64/iommu/iommu.h Mon Nov 16 21:55:52 2020 (r367736) @@ -0,0 +1,44 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2020 Ruslan Bukin <b...@bsdpad.com> + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory (Department of Computer Science and + * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the + * DARPA SSITH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARM64_IOMMU_IOMMU_H_ +#define _ARM64_IOMMU_IOMMU_H_ + +#define IOMMU_PAGE_SIZE 4096 +#define IOMMU_PAGE_MASK (IOMMU_PAGE_SIZE - 1) + +int iommu_unregister(struct iommu_unit *unit); +int iommu_register(struct iommu_unit *unit); + +#endif /* _ARM64_IOMMU_IOMMU_H_ */ Added: head/sys/arm64/iommu/iommu_if.m ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/arm64/iommu/iommu_if.m Mon Nov 16 21:55:52 2020 (r367736) @@ -0,0 +1,125 @@ +#- +# SPDX-License-Identifier: BSD-2-Clause +#: +# Copyright (c) 2020 Ruslan Bukin <b...@bsdpad.com> +# +# This software was developed by SRI International and the University of +# Cambridge Computer Laboratory (Department of Computer Science and +# Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the +# DARPA SSITH research programme. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ +# + +#include <sys/types.h> +#include <sys/taskqueue.h> +#include <sys/bus.h> +#include <sys/sysctl.h> +#include <sys/tree.h> +#include <vm/vm.h> +#include <dev/pci/pcireg.h> +#include <dev/pci/pcivar.h> +#include <dev/iommu/iommu.h> + +INTERFACE iommu; + +# +# Check if the iommu controller dev is responsible to serve traffic +# for a given child. +# +METHOD int find { + device_t dev; + device_t child; +}; + +# +# Map a virtual address VA to a physical address PA. +# +METHOD int map { + device_t dev; + struct iommu_domain *iodom; + vm_offset_t va; + vm_page_t *ma; + bus_size_t size; + vm_prot_t prot; +}; + +# +# Unmap a virtual address VA. +# +METHOD int unmap { + device_t dev; + struct iommu_domain *iodom; + vm_offset_t va; + bus_size_t size; +}; + +# +# Allocate an IOMMU domain. +# +METHOD struct iommu_domain * domain_alloc { + device_t dev; + struct iommu_unit *iommu; +}; + +# +# Release all the resources held by IOMMU domain. +# +METHOD void domain_free { + device_t dev; + struct iommu_domain *iodom; +}; + +# +# Find a domain allocated for a dev. +# +METHOD struct iommu_domain * domain_lookup { + device_t dev; +}; + +# +# Find an allocated context for a device. +# +METHOD struct iommu_ctx * ctx_lookup { + device_t dev; + device_t child; +}; + +# +# Allocate a new iommu context. +# +METHOD struct iommu_ctx * ctx_alloc { + device_t dev; + struct iommu_domain *iodom; + device_t child; + bool disabled; +}; + +# +# Free the iommu context. +# +METHOD void ctx_free { + device_t dev; + struct iommu_ctx *ioctx; +}; Added: head/sys/arm64/iommu/smmu.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/arm64/iommu/smmu.c Mon Nov 16 21:55:52 2020 (r367736) @@ -0,0 +1,1937 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2019-2020 Ruslan Bukin <b...@bsdpad.com> + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory (Department of Computer Science and + * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the + * DARPA SSITH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Hardware overview. + * + * An incoming transaction from a peripheral device has an address, size, + * attributes and StreamID. + * + * In case of PCI-based devices, StreamID is a PCI rid. + * + * The StreamID is used to select a Stream Table Entry (STE) in a Stream table, + * which contains per-device configuration. + * + * Stream table is a linear or 2-level walk table (this driver supports both). + * Note that a linear table could occupy 1GB or more of memory depending on + * sid_bits value. + * + * STE is used to locate a Context Descriptor, which is a struct in memory + * that describes stages of translation, translation table type, pointer to + * level 0 of page tables, ASID, etc. + * + * Hardware supports two stages of translation: Stage1 (S1) and Stage2 (S2): + * o S1 is used for the host machine traffic translation + * o S2 is for a hypervisor + * + * This driver enables S1 stage with standard AArch64 page tables. + * + * Note that SMMU does not share TLB with a main CPU. + * Command queue is used by this driver to Invalidate SMMU TLB, STE cache. + * + * An arm64 SoC could have more than one SMMU instance. + * ACPI IORT table describes which SMMU unit is assigned for a particular + * peripheral device. + * + * Queues. + * + * Register interface and Memory-based circular buffer queues are used + * to inferface SMMU. + * + * These are a Command queue for commands to send to the SMMU and an Event + * queue for event/fault reports from the SMMU. Optionally PRI queue is + * designed for PCIe page requests reception. + * + * Note that not every hardware supports PRI services. For instance they were + * not found in Neoverse N1 SDP machine. + * (This drivers does not implement PRI queue.) + * + * All SMMU queues are arranged as circular buffers in memory. They are used + * in a producer-consumer fashion so that an output queue contains data + * produced by the SMMU and consumed by software. + * An input queue contains data produced by software, consumed by the SMMU. + * + * Interrupts. + * + * Interrupts are not required by this driver for normal operation. + * The standard wired interrupt is only triggered when an event comes from + * the SMMU, which is only in a case of errors (e.g. translation fault). + */ + +#include "opt_platform.h" +#include "opt_acpi.h" + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/bitstring.h> +#include <sys/bus.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/rman.h> +#include <sys/lock.h> +#include <sys/tree.h> +#include <sys/taskqueue.h> +#include <vm/vm.h> +#include <vm/vm_page.h> +#if DEV_ACPI +#include <contrib/dev/acpica/include/acpi.h> +#include <dev/acpica/acpivar.h> +#endif +#include <dev/pci/pcireg.h> +#include <dev/pci/pcivar.h> +#include <dev/iommu/iommu.h> + +#include "iommu.h" +#include "iommu_if.h" + +#include "smmureg.h" +#include "smmuvar.h" + +#define STRTAB_L1_SZ_SHIFT 20 +#define STRTAB_SPLIT 8 + +#define STRTAB_L1_DESC_L2PTR_M (0x3fffffffffff << 6) +#define STRTAB_L1_DESC_DWORDS 1 + +#define STRTAB_STE_DWORDS 8 + +#define CMDQ_ENTRY_DWORDS 2 +#define EVTQ_ENTRY_DWORDS 4 +#define PRIQ_ENTRY_DWORDS 2 + +#define CD_DWORDS 8 + +#define Q_WRP(q, p) ((p) & (1 << (q)->size_log2)) +#define Q_IDX(q, p) ((p) & ((1 << (q)->size_log2) - 1)) +#define Q_OVF(p) ((p) & (1 << 31)) /* Event queue overflowed */ + +#define SMMU_Q_ALIGN (64 * 1024) + +static struct resource_spec smmu_spec[] = { + { SYS_RES_MEMORY, 0, RF_ACTIVE }, + { SYS_RES_IRQ, 0, RF_ACTIVE }, + { SYS_RES_IRQ, 1, RF_ACTIVE }, + { SYS_RES_IRQ, 2, RF_ACTIVE }, + RESOURCE_SPEC_END +}; + +MALLOC_DEFINE(M_SMMU, "SMMU", SMMU_DEVSTR); + +#define dprintf(fmt, ...) + +struct smmu_event { + int ident; + char *str; + char *msg; +}; + +static struct smmu_event events[] = { + { 0x01, "F_UUT", + "Unsupported Upstream Transaction."}, + { 0x02, "C_BAD_STREAMID", + "Transaction StreamID out of range."}, + { 0x03, "F_STE_FETCH", + "Fetch of STE caused external abort."}, + { 0x04, "C_BAD_STE", + "Used STE invalid."}, + { 0x05, "F_BAD_ATS_TREQ", + "Address Translation Request disallowed for a StreamID " + "and a PCIe ATS Translation Request received."}, + { 0x06, "F_STREAM_DISABLED", + "The STE of a transaction marks non-substream transactions " + "disabled."}, + { 0x07, "F_TRANSL_FORBIDDEN", + "An incoming PCIe transaction is marked Translated but " + "SMMU bypass is disallowed for this StreamID."}, + { 0x08, "C_BAD_SUBSTREAMID", + "Incoming SubstreamID present, but configuration is invalid."}, + { 0x09, "F_CD_FETCH", + "Fetch of CD caused external abort."}, + { 0x0a, "C_BAD_CD", + "Fetched CD invalid."}, + { 0x0b, "F_WALK_EABT", + "An external abort occurred fetching (or updating) " + "a translation table descriptor."}, + { 0x10, "F_TRANSLATION", + "Translation fault."}, + { 0x11, "F_ADDR_SIZE", + "Address Size fault."}, + { 0x12, "F_ACCESS", + "Access flag fault due to AF == 0 in a page or block TTD."}, + { 0x13, "F_PERMISSION", + "Permission fault occurred on page access."}, + { 0x20, "F_TLB_CONFLICT", + "A TLB conflict occurred because of the transaction."}, + { 0x21, "F_CFG_CONFLICT", + "A configuration cache conflict occurred due to " + "the transaction."}, + { 0x24, "E_PAGE_REQUEST", + "Speculative page request hint."}, + { 0x25, "F_VMS_FETCH", + "Fetch of VMS caused external abort."}, + { 0, NULL, NULL }, +}; + +static int +smmu_q_has_space(struct smmu_queue *q) +{ + + /* + * See 6.3.27 SMMU_CMDQ_PROD + * + * There is space in the queue for additional commands if: + * SMMU_CMDQ_CONS.RD != SMMU_CMDQ_PROD.WR || + * SMMU_CMDQ_CONS.RD_WRAP == SMMU_CMDQ_PROD.WR_WRAP + */ + + if (Q_IDX(q, q->lc.cons) != Q_IDX(q, q->lc.prod) || + Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod)) + return (1); + + return (0); +} + +static int +smmu_q_empty(struct smmu_queue *q) +{ + + if (Q_IDX(q, q->lc.cons) == Q_IDX(q, q->lc.prod) && + Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod)) + return (1); + + return (0); +} + +static int __unused +smmu_q_consumed(struct smmu_queue *q, uint32_t prod) +{ + + if ((Q_WRP(q, q->lc.cons) == Q_WRP(q, prod)) && + (Q_IDX(q, q->lc.cons) >= Q_IDX(q, prod))) + return (1); + + if ((Q_WRP(q, q->lc.cons) != Q_WRP(q, prod)) && + (Q_IDX(q, q->lc.cons) <= Q_IDX(q, prod))) + return (1); + + return (0); +} + +static uint32_t +smmu_q_inc_cons(struct smmu_queue *q) +{ + uint32_t cons; + uint32_t val; + + cons = (Q_WRP(q, q->lc.cons) | Q_IDX(q, q->lc.cons)) + 1; + val = (Q_OVF(q->lc.cons) | Q_WRP(q, cons) | Q_IDX(q, cons)); + + return (val); +} + +static uint32_t +smmu_q_inc_prod(struct smmu_queue *q) +{ + uint32_t prod; + uint32_t val; + + prod = (Q_WRP(q, q->lc.prod) | Q_IDX(q, q->lc.prod)) + 1; + val = (Q_OVF(q->lc.prod) | Q_WRP(q, prod) | Q_IDX(q, prod)); + + return (val); +} + +static int +smmu_write_ack(struct smmu_softc *sc, uint32_t reg, + uint32_t reg_ack, uint32_t val) +{ + uint32_t v; + int timeout; + + timeout = 100000; + + bus_write_4(sc->res[0], reg, val); + + do { + v = bus_read_4(sc->res[0], reg_ack); + if (v == val) + break; + } while (timeout--); + + if (timeout <= 0) { + device_printf(sc->dev, "Failed to write reg.\n"); + return (-1); + } + + return (0); +} + +static inline int +ilog2(long x) +{ + + KASSERT(x > 0 && powerof2(x), ("%s: invalid arg %ld", __func__, x)); + + return (flsl(x) - 1); +} + +static int +smmu_init_queue(struct smmu_softc *sc, struct smmu_queue *q, + uint32_t prod_off, uint32_t cons_off, uint32_t dwords) +{ + int sz; + + sz = (1 << q->size_log2) * dwords * 8; + + /* Set up the command circular buffer */ + q->vaddr = contigmalloc(sz, M_SMMU, + M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, SMMU_Q_ALIGN, 0); + if (q->vaddr == NULL) { + device_printf(sc->dev, "failed to allocate %d bytes\n", sz); + return (-1); + } + + q->prod_off = prod_off; + q->cons_off = cons_off; + q->paddr = vtophys(q->vaddr); + + q->base = CMDQ_BASE_RA | EVENTQ_BASE_WA | PRIQ_BASE_WA; + q->base |= q->paddr & Q_BASE_ADDR_M; + q->base |= q->size_log2 << Q_LOG2SIZE_S; + + return (0); +} + +static int +smmu_init_queues(struct smmu_softc *sc) +{ + int err; + + /* Command queue. */ + err = smmu_init_queue(sc, &sc->cmdq, + SMMU_CMDQ_PROD, SMMU_CMDQ_CONS, CMDQ_ENTRY_DWORDS); + if (err) + return (ENXIO); + + /* Event queue. */ + err = smmu_init_queue(sc, &sc->evtq, + SMMU_EVENTQ_PROD, SMMU_EVENTQ_CONS, EVTQ_ENTRY_DWORDS); + if (err) + return (ENXIO); + + if (!(sc->features & SMMU_FEATURE_PRI)) + return (0); + + /* PRI queue. */ + err = smmu_init_queue(sc, &sc->priq, + SMMU_PRIQ_PROD, SMMU_PRIQ_CONS, PRIQ_ENTRY_DWORDS); + if (err) + return (ENXIO); + + return (0); +} + +/* + * Dump 2LVL or linear STE. + */ +static void +smmu_dump_ste(struct smmu_softc *sc, int sid) +{ + struct smmu_strtab *strtab; + struct l1_desc *l1_desc; + uint64_t *ste, *l1; + int i; + + strtab = &sc->strtab; + + if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { + i = sid >> STRTAB_SPLIT; + l1 = (void *)((uint64_t)strtab->vaddr + + STRTAB_L1_DESC_DWORDS * 8 * i); + device_printf(sc->dev, "L1 ste == %lx\n", l1[0]); + + l1_desc = &strtab->l1[i]; + ste = l1_desc->va; + if (ste == NULL) /* L2 is not initialized */ + return; + } else { + ste = (void *)((uint64_t)strtab->vaddr + + sid * (STRTAB_STE_DWORDS << 3)); + } + + /* Dump L2 or linear STE. */ + for (i = 0; i < STRTAB_STE_DWORDS; i++) + device_printf(sc->dev, "ste[%d] == %lx\n", i, ste[i]); +} + +static void __unused +smmu_dump_cd(struct smmu_softc *sc, struct smmu_cd *cd) +{ + uint64_t *vaddr; + int i; + + device_printf(sc->dev, "%s\n", __func__); + + vaddr = cd->vaddr; + for (i = 0; i < CD_DWORDS; i++) + device_printf(sc->dev, "cd[%d] == %lx\n", i, vaddr[i]); *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
