> Date: Thu, 8 Oct 2020 00:34:28 -0500
> From: Jordan Hargrave <jordan_hargr...@hotmail.com>
>
> Ok updated the new changes.
I think this is good enough for further cleanup in the tree now.
Builds all thr amd64 kernels, doesn't break i386 and arm64 GENERIC.MP.
However, I think acpidmar(4) shouldn't be enabled yet until have done
a bit more testing. Theo do we want to compile in the driver such
that people can easily flip it on in UKC?
I also noticed that re(4) does a bad DMA transfer as soon as the
interface is brought up. With your diff the kernel doesn't panic if
that happensl it just spits out some debugging information. We may
want to change that in the future.
ok kettenis@ once the question about enabling acpidmar(4) is resolved.
>
> On Mon, Oct 05, 2020 at 09:54:02PM +0200, Mark Kettenis wrote:
> > > Date: Thu, 17 Sep 2020 20:54:51 -0500
> > > From: Jordan Hargrave <jordan_hargr...@hotmail.com>
> > > Cc: ma...@peereboom.org, kette...@openbsd.org, tech@openbsd.org,
> > > d...@openbsd.org, j...@openbsd.org
> > > Content-Type: text/plain; charset=us-ascii
> > > Content-Disposition: inline
> > >
> > > Ok made more changes....
> > >
> > > >
> > > > Should be handled by that activate function as well.
> > > >
> >
> > So there are quite a few style issues. I can point them out to you,
> > or I could fix them after this is committed, which is probably more
> > efficient.
> >
> > Also, there seems to be lot of debug code left in here that should be
> > removed or at least hidden before this gets committed.
> >
> > >
> > > diff --git a/sys/arch/amd64/conf/GENERIC b/sys/arch/amd64/conf/GENERIC
> > > index 1d6397391..a69c72c26 100644
> > > --- a/sys/arch/amd64/conf/GENERIC
> > > +++ b/sys/arch/amd64/conf/GENERIC
> > > @@ -45,6 +45,7 @@ acpibtn* at acpi?
> > > acpicpu* at acpi?
> > > acpicmos* at acpi?
> > > acpidock* at acpi?
> > > +acpidmar0 at acpi?
> > > acpiec* at acpi?
> > > acpipci* at acpi?
> > > acpiprt* at acpi?
> > > diff --git a/sys/arch/amd64/include/pci_machdep.h
> > > b/sys/arch/amd64/include/pci_machdep.h
> > > index bc295cc22..ea09f1abc 100644
> > > --- a/sys/arch/amd64/include/pci_machdep.h
> > > +++ b/sys/arch/amd64/include/pci_machdep.h
> > > @@ -91,7 +91,12 @@ void
> > > *pci_intr_establish_cpu(pci_chipset_tag_t, pci_intr_handle_t,
> > > int, struct cpu_info *,
> > > int (*)(void *), void *, const char *);
> > > void pci_intr_disestablish(pci_chipset_tag_t, void *);
> > > +#if NACPIDMAR > 0
> > > +int pci_probe_device_hook(pci_chipset_tag_t,
> > > + struct pci_attach_args *);
> > > +#else
> > > #define pci_probe_device_hook(c, a) (0)
> > > +#endif
> >
> > This is probably a bad idea. You don't include "acpidmar.h" in this
> > file, and doing so is probaly undesireable. But that means the
> > definition of the hook function depends on whether the file that
> > includes this does that or not.
> >
> > Better just unconditionally provide the prototype and use a #if
> > NACPIDMAR > 0 in the implementation.
> >
>
> Ok changed to that method
>
> > > +#include "acpidmar.h"
> > > +#include "amd_iommu.h"
> > > +
> > > +//#define IOMMU_DEBUG
> >
> > No C++-style comments please. Make this an #undef or use /* */.
> >
> > > +
> > > +#ifdef IOMMU_DEBUG
> > > +#define dprintf(x...) printf(x)
> > > +#else
> > > +#define dprintf(x...)
> > > +#endif
> > > +
> > > +#ifdef DDB
> > > +int acpidmar_ddb = 0;
> > > +#endif
> > > +
> > > +int intel_iommu_gfx_mapped = 0;
> >
> > Unused variable.
> >
> > > +int force_cm = 1;
> >
> > Rename to "acpidmar_force_cm"?
> >
> > > +
> > > +void showahci(void *);
> >
> > Unused prototype.
> >
> > > +
> > > +/* Page Table Entry per domain */
> > > +struct iommu_softc;
> > > +
> > > +static inline int
> > > +mksid(int b, int d, int f)
> > > +{
> > > + return (b << 8) + (d << 3) + f;
> > > +}
> > > +
> > > +static inline int
> > > +sid_devfn(int sid)
> > > +{
> > > + return sid & 0xff;
> > > +}
> > > +
> > > +static inline int
> > > +sid_bus(int sid)
> > > +{
> > > + return (sid >> 8) & 0xff;
> > > +}
> > > +
> > > +static inline int
> > > +sid_dev(int sid)
> > > +{
> > > + return (sid >> 3) & 0x1f;
> > > +}
> > > +
> > > +static inline int
> > > +sid_fun(int sid)
> > > +{
> > > + return (sid >> 0) & 0x7;
> > > +}
> > > +
> > > +/* Alias mapping */
> > > +#define SID_INVALID 0x80000000L
> > > +static uint32_t sid_flag[65536];
> > > +
> > > +struct domain_dev {
> > > + int sid;
> > > + int sec;
> > > + int sub;
> > > + TAILQ_ENTRY(domain_dev) link;
> > > +};
> > > +
> > > +struct domain {
> > > + struct iommu_softc *iommu;
> > > + int did;
> > > + int gaw;
> > > + struct pte_entry *pte;
> > > + paddr_t ptep;
> > > + struct bus_dma_tag dmat;
> > > + int flag;
> > > +
> > > + struct mutex exlck;
> > > + char exname[32];
> > > + struct extent *iovamap;
> > > + TAILQ_HEAD(,domain_dev) devices;
> > > + TAILQ_ENTRY(domain) link;
> > > +};
> > > +
> > > +#define DOM_DEBUG 0x1
> > > +#define DOM_NOMAP 0x2
> > > +
> > > +struct dmar_devlist {
> > > + int type;
> > > + int bus;
> > > + int ndp;
> > > + struct acpidmar_devpath *dp;
> > > + TAILQ_ENTRY(dmar_devlist) link;
> > > +};
> > > +
> > > +TAILQ_HEAD(devlist_head, dmar_devlist);
> > > +
> > > +struct ivhd_devlist {
> > > + int start_id;
> > > + int end_id;
> > > + int cfg;
> > > + TAILQ_ENTRY(ivhd_devlist) link;
> > > +};
> > > +
> > > +struct rmrr_softc {
> > > + TAILQ_ENTRY(rmrr_softc) link;
> > > + struct devlist_head devices;
> > > + int segment;
> > > + uint64_t start;
> > > + uint64_t end;
> > > +};
> > > +
> > > +struct atsr_softc {
> > > + TAILQ_ENTRY(atsr_softc) link;
> > > + struct devlist_head devices;
> > > + int segment;
> > > + int flags;
> > > +};
> > > +
> > > +struct iommu_pic {
> > > + struct pic pic;
> > > + struct iommu_softc *iommu;
> > > +};
> > > +
> > > +#define IOMMU_FLAGS_CATCHALL 0x1
> > > +#define IOMMU_FLAGS_BAD 0x2
> > > +#define IOMMU_FLAGS_SUSPEND 0x4
> > > +
> > > +struct iommu_softc {
> > > + TAILQ_ENTRY(iommu_softc)link;
> > > + struct devlist_head devices;
> > > + int id;
> > > + int flags;
> > > + int segment;
> > > +
> > > + struct mutex reg_lock;
> > > +
> > > + bus_space_tag_t iot;
> > > + bus_space_handle_t ioh;
> > > +
> > > + uint64_t cap;
> > > + uint64_t ecap;
> > > + uint32_t gcmd;
> > > +
> > > + int mgaw;
> > > + int agaw;
> > > + int ndoms;
> > > +
> > > + struct root_entry *root;
> > > + struct context_entry *ctx[256];
> > > +
> > > + void *intr;
> > > + struct iommu_pic pic;
> > > + int fedata;
> > > + uint64_t feaddr;
> > > + uint64_t rtaddr;
> > > +
> > > + // Queued Invalidation
> >
> > C++ comment.
> >
> > > + int qi_head;
> > > + int qi_tail;
> > > + paddr_t qip;
> > > + struct qi_entry *qi;
> > > +
> > > + struct domain *unity;
> > > + TAILQ_HEAD(,domain) domains;
> > > +}
> > > +
> > > +/* Intel: Initialize IOMMU */
> > > +int
> > > +iommu_init(struct acpidmar_softc *sc, struct iommu_softc *iommu,
> > > + struct acpidmar_drhd *dh)
> > > +{
> > > + static int niommu;
> > > + int len = VTD_PAGE_SIZE;
> > > + int i, gaw;
> > > + uint32_t sts;
> > > + paddr_t paddr;
> > > +
> > > + if (_bus_space_map(sc->sc_memt, dh->address, len, 0, &iommu->ioh) != 0)
> > > {
> > > + return (-1);
> > > + }
> >
> > Why are you using _bus_space_map() here? Do you expect this address
> > space to be mapped by some other driver?
> >
>
> It shouldn't be used anywhere else.
>
> > > +
> > > + TAILQ_INIT(&iommu->domains);
> > > + iommu->id = ++niommu;
> > > + iommu->flags = dh->flags;
> > > + iommu->segment = dh->segment;
> > > + iommu->iot = sc->sc_memt;
> > > +
> > > + iommu->cap = iommu_read_8(iommu, DMAR_CAP_REG);
> > > + iommu->ecap = iommu_read_8(iommu, DMAR_ECAP_REG);
> > > + iommu->ndoms = cap_nd(iommu->cap);
> > > +
> > > + printf(" caps: %s%s%s%s%s%s%s%s%s%s%s\n",
> > > + iommu->cap & CAP_AFL ? "afl " : "", // adv fault
> > > + iommu->cap & CAP_RWBF ? "rwbf " : "", // write-buffer flush
> > > + iommu->cap & CAP_PLMR ? "plmr " : "", // protected lo region
> > > + iommu->cap & CAP_PHMR ? "phmr " : "", // protected hi region
> > > + iommu->cap & CAP_CM ? "cm " : "", // caching mode
> > > + iommu->cap & CAP_ZLR ? "zlr " : "", // zero-length read
> > > + iommu->cap & CAP_PSI ? "psi " : "", // page invalidate
> > > + iommu->cap & CAP_DWD ? "dwd " : "", // write drain
> > > + iommu->cap & CAP_DRD ? "drd " : "", // read drain
> > > + iommu->cap & CAP_FL1GP ? "Gb " : "", // 1Gb pages
> > > + iommu->cap & CAP_PI ? "pi " : ""); // posted interrupts
> > > + printf(" ecap: %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
> > > + iommu->ecap & ECAP_C ? "c " : "", // coherent
> > > + iommu->ecap & ECAP_QI ? "qi " : "", // queued invalidate
> > > + iommu->ecap & ECAP_DT ? "dt " : "", // device iotlb
> > > + iommu->ecap & ECAP_IR ? "ir " : "", // intr remap
> > > + iommu->ecap & ECAP_EIM ? "eim " : "", // x2apic
> > > + iommu->ecap & ECAP_PT ? "pt " : "", // passthrough
> > > + iommu->ecap & ECAP_SC ? "sc " : "", // snoop control
> > > + iommu->ecap & ECAP_ECS ? "ecs " : "", // extended context
> > > + iommu->ecap & ECAP_MTS ? "mts " : "", // memory type
> > > + iommu->ecap & ECAP_NEST ? "nest " : "", // nested translations
> > > + iommu->ecap & ECAP_DIS ? "dis " : "", // deferred invalidation
> > > + iommu->ecap & ECAP_PASID ? "pas " : "", // pasid
> > > + iommu->ecap & ECAP_PRS ? "prs " : "", // page request
> > > + iommu->ecap & ECAP_ERS ? "ers " : "", // execute request
> > > + iommu->ecap & ECAP_SRS ? "srs " : "", // supervisor request
> > > + iommu->ecap & ECAP_NWFS ? "nwfs " : "", // no write flag
> > > + iommu->ecap & ECAP_EAFS ? "eafs " : ""); // extended accessed
> > > flag
> >
> > Are these debug printfs? Or are you planning to keep these?
> >
>
> Made them all debugs
>
> > > +
> > > + mtx_init(&iommu->reg_lock, IPL_HIGH);
> > > +
> > > + /* Clear Interrupt Masking */
> > > + iommu_write_4(iommu, DMAR_FSTS_REG, FSTS_PFO | FSTS_PPF);
> > > +
> > > + iommu->intr = acpidmar_intr_establish(iommu, IPL_HIGH,
> > > + acpidmar_intr, iommu, "dmarintr");
> > > +
> > > + /* Enable interrupts */
> > > + sts = iommu_read_4(iommu, DMAR_FECTL_REG);
> > > + iommu_write_4(iommu, DMAR_FECTL_REG, sts & ~FECTL_IM);
> > > +
> > > + /* Allocate root pointer */
> > > + iommu->root = iommu_alloc_page(iommu, &paddr);
> > > +#ifdef DEBUG
> > > + printf("Allocated root pointer: pa:%.16llx va:%p\n",
> > > + (uint64_t)paddr, iommu->root);
> > > +#endif
> >
> > Probably should make this #ifdef IOMMU_DEBUG
> >
> > > + iommu->rtaddr = paddr;
> > > + iommu_flush_write_buffer(iommu);
> > > + iommu_set_rtaddr(iommu, paddr);
> > > +
> > > +#if 0
> > > + if (iommu->ecap & ECAP_QI) {
> > > + /* Queued Invalidation support */
> > > + iommu->qi = iommu_alloc_page(iommu, &iommu->qip);
> > > + iommu_write_8(iommu, DMAR_IQT_REG, 0);
> > > + iommu_write_8(iommu, DMAR_IQA_REG, iommu->qip | IQA_QS_256);
> > > + }
> > > + if (iommu->ecap & ECAP_IR) {
> > > + /* Interrupt remapping support */
> > > + iommu_write_8(iommu, DMAR_IRTA_REG, 0);
> > > + }
> > > +#endif
> >
> > At some point you'll need to decide whether this code should be kept
> > or not...
> >
> > > +
> > > + /* Calculate guest address width and supported guest widths */
> > > + gaw = -1;
> > > + iommu->mgaw = cap_mgaw(iommu->cap);
> > > + printf("gaw: %d { ", iommu->mgaw);
> > > + for (i = 0; i < 5; i++) {
> > > + if (cap_sagaw(iommu->cap) & (1L << i)) {
> > > + gaw = VTD_LEVELTOAW(i);
> > > + printf("%d ", gaw);
> > > + iommu->agaw = gaw;
> > > + }
> > > + }
> > > + printf("}\n");
> >
> > More debug printfs?
> >
> > > +
> > > + /* Cache current status register bits */
> > > + sts = iommu_read_4(iommu, DMAR_GSTS_REG);
> > > + if (sts & GSTS_TES)
> > > + iommu->gcmd |= GCMD_TE;
> > > + if (sts & GSTS_QIES)
> > > + iommu->gcmd |= GCMD_QIE;
> > > + if (sts & GSTS_IRES)
> > > + iommu->gcmd |= GCMD_IRE;
> > > + if (iommu->gcmd) {
> > > + printf("gcmd: %x preset\n", iommu->gcmd);
> > > + }
> > > + acpidmar_intr(iommu);
> > > + return (0);
> > > +}
> > > +
> > > +const char *dmar_rn(int reg);
> > > +
> > > +const char *
> > > +dmar_rn(int reg)
> > > +{
> > > + switch (reg) {
> > > + case EVT_HEAD_REG: return "evthead";
> > > + case EVT_TAIL_REG: return "evttail";
> > > + case CMD_HEAD_REG: return "cmdhead";
> > > + case CMD_TAIL_REG: return "cmdtail";
> > > + case CMD_BASE_REG: return "cmdbase";
> > > + case EVT_BASE_REG: return "evtbase";
> > > + case DEV_TAB_BASE_REG: return "devtblbase";
> > > + case IOMMUCTL_REG: return "iommuctl";
> > > +#if 0
> > > + case DMAR_VER_REG: return "ver";
> > > + case DMAR_CAP_REG: return "cap";
> > > + case DMAR_ECAP_REG: return "ecap";
> > > + case DMAR_GSTS_REG: return "gsts";
> > > + case DMAR_GCMD_REG: return "gcmd";
> > > + case DMAR_FSTS_REG: return "fsts";
> > > + case DMAR_FECTL_REG: return "fectl";
> > > + case DMAR_RTADDR_REG: return "rtaddr";
> > > + case DMAR_FEDATA_REG: return "fedata";
> > > + case DMAR_FEADDR_REG: return "feaddr";
> > > + case DMAR_FEUADDR_REG: return "feuaddr";
> > > + case DMAR_PMEN_REG: return "pmen";
> > > + case DMAR_IEDATA_REG: return "iedata";
> > > + case DMAR_IEADDR_REG: return "ieaddr";
> > > + case DMAR_IEUADDR_REG: return "ieuaddr";
> > > + case DMAR_IRTA_REG: return "irta";
> > > + case DMAR_CCMD_REG: return "ccmd";
> > > + case DMAR_IQH_REG: return "iqh";
> > > + case DMAR_IQT_REG: return "iqt";
> > > + case DMAR_IQA_REG: return "iqa";
> > > +#endif
> > > + }
> > > + return "unknown";
> > > +}
> > > +
> > > +/* Read/Write IOMMU register */
> > > +uint32_t
> > > +iommu_read_4(struct iommu_softc *iommu, int reg)
> > > +{
> > > + uint32_t v;
> > > +
> > > + v = bus_space_read_4(iommu->iot, iommu->ioh, reg);
> > > + if (reg < 00) {
> > > + printf("iommu%d: read %x %.8lx [%s]\n",
> > > + iommu->id, reg, (unsigned long)v, dmar_rn(reg));
> > > + }
> > > +
> > > + return (v);
> > > +}
> > > +
> > > +
> > > +#define dbprintf(x...)
> >
> > What is the point of this? Shoulkd this be a IOMMU_DEBUG feature?
> >
> > > +
> > > +void
> > > +iommu_write_4(struct iommu_softc *iommu, int reg, uint32_t v)
> > > +{
> > > + dbprintf("iommu%d: write %.8x %.16lx [%s]\n",
> > > + iommu->id, reg, (unsigned long)v, dmar_rn(reg));
> > > + bus_space_write_4(iommu->iot, iommu->ioh, reg, (uint32_t)v);
> > > +}
> > > +
> > > +uint64_t
> > > +iommu_read_8(struct iommu_softc *iommu, int reg)
> > > +{
> > > + uint64_t v;
> > > +
> > > + v = bus_space_read_8(iommu->iot, iommu->ioh, reg);
> > > + if (reg < 00) {
> >
> > double-nul? In any case, checking for reg < 0 doesn't make sense to me...
> >
> > > + printf("iommu%d: read %x %.8lx [%s]\n",
> > > + iommu->id, reg, (unsigned long)v, dmar_rn(reg));
> > > + }
> > > +
> > > + return (v);
> > > +}
> > > +
> > > +void
> > > +iommu_write_8(struct iommu_softc *iommu, int reg, uint64_t v)
> > > +{
> > > + dbprintf("iommu%d: write %.8x %.16lx [%s]\n",
> > > + iommu->id, reg, (unsigned long)v, dmar_rn(reg));
> > > + bus_space_write_8(iommu->iot, iommu->ioh, reg, v);
> > > +}
> > > +
> > > +/* Check if a device is within a device scope */
> > > +int
> > > +acpidmar_match_devscope(struct devlist_head *devlist, pci_chipset_tag_t
> > > pc,
> > > + int sid)
> > > +{
> > > + struct dmar_devlist *ds;
> > > + int sub, sec, i;
> > > + int bus, dev, fun, sbus;
> > > + pcireg_t reg;
> > > + pcitag_t tag;
> > > +
> > > + sbus = sid_bus(sid);
> > > + TAILQ_FOREACH(ds, devlist, link) {
> > > + bus = ds->bus;
> > > + dev = ds->dp[0].device;
> > > + fun = ds->dp[0].function;
> > > + /* Walk PCI bridges in path */
> > > + for (i = 1; i < ds->ndp; i++) {
> > > + tag = pci_make_tag(pc, bus, dev, fun);
> > > + reg = pci_conf_read(pc, tag, PPB_REG_BUSINFO);
> > > + bus = PPB_BUSINFO_SECONDARY(reg);
> > > + dev = ds->dp[i].device;
> > > + fun = ds->dp[i].function;
> > > + }
> > > +
> > > + /* Check for device exact match */
> > > + if (sid == mksid(bus, dev, fun)) {
> > > + return DMAR_ENDPOINT;
> > > + }
> > > +
> > > + /* Check for device subtree match */
> > > + if (ds->type == DMAR_BRIDGE) {
> > > + tag = pci_make_tag(pc, bus, dev, fun);
> > > + reg = pci_conf_read(pc, tag, PPB_REG_BUSINFO);
> > > + sec = PPB_BUSINFO_SECONDARY(reg);
> > > + sub = PPB_BUSINFO_SUBORDINATE(reg);
> > > + if (sec <= sbus && sbus <= sub) {
> > > + return DMAR_BRIDGE;
> > > + }
> > > + }
> > > + }
> > > +
> > > + return (0);
> > > +}
> > > +
> > > +struct domain *
> > > +domain_create(struct iommu_softc *iommu, int did)
> > > +{
> > > + struct domain *dom;
> > > + int gaw;
> > > +
> > > + dprintf("iommu%d: create domain: %.4x\n", iommu->id, did);
> > > + dom = malloc(sizeof(*dom), M_DEVBUF, M_ZERO | M_WAITOK);
> > > + dom->did = did;
> > > + dom->iommu = iommu;
> > > + dom->pte = iommu_alloc_page(iommu, &dom->ptep);
> > > + TAILQ_INIT(&dom->devices);
> > > +
> > > + /* Setup DMA */
> > > + dom->dmat._cookie = dom;
> > > + dom->dmat._dmamap_create = dmar_dmamap_create; // nop
> > > + dom->dmat._dmamap_destroy = dmar_dmamap_destroy; // nop
> > > + dom->dmat._dmamap_load = dmar_dmamap_load; // lm
> > > + dom->dmat._dmamap_load_mbuf = dmar_dmamap_load_mbuf; // lm
> > > + dom->dmat._dmamap_load_uio = dmar_dmamap_load_uio; // lm
> > > + dom->dmat._dmamap_load_raw = dmar_dmamap_load_raw; // lm
> > > + dom->dmat._dmamap_unload = dmar_dmamap_unload; // um
> > > + dom->dmat._dmamap_sync = dmar_dmamap_sync; // lm
> > > + dom->dmat._dmamem_alloc = dmar_dmamem_alloc; // nop
> > > + dom->dmat._dmamem_free = dmar_dmamem_free; // nop
> > > + dom->dmat._dmamem_map = dmar_dmamem_map; // nop
> > > + dom->dmat._dmamem_unmap = dmar_dmamem_unmap; // nop
> > > + dom->dmat._dmamem_mmap = dmar_dmamem_mmap;
> > > +
> > > + snprintf(dom->exname, sizeof(dom->exname), "did:%x.%.4x",
> > > + iommu->id, dom->did);
> > > +
> > > + /* Setup IOMMU address map */
> > > + gaw = min(iommu->agaw, iommu->mgaw);
> > > + dom->iovamap = extent_create(dom->exname, 1024*1024*16,
> > > + (1LL << gaw)-1,
> > > + M_DEVBUF, NULL, 0,
> > > + EX_WAITOK|EX_NOCOALESCE);
> > > +
> > > + /* Zero out Interrupt region */
> > > + extent_alloc_region(dom->iovamap, 0xFEE00000L, 0x100000,
> > > + EX_WAITOK);
> >
> > Magic hardcoded address?
> >
>
> It's the MSI interrupt address. Linux reserves those areas so I marked them
> invalid as well.
>
> > > + mtx_init(&dom->exlck, IPL_HIGH);
> > > +
> > > + TAILQ_INSERT_TAIL(&iommu->domains, dom, link);
> > > +
> > > + return dom;
> > > +}
> > > +
> > > +void
> > > +domain_add_device(struct domain *dom, int sid)
> > > +{
> > > + struct domain_dev *ddev;
> > > +
> > > + dprintf("add %s to iommu%d.%.4x\n", dmar_bdf(sid), dom->iommu->id,
> > > dom->did);
> > > + ddev = malloc(sizeof(*ddev), M_DEVBUF, M_ZERO | M_WAITOK);
> > > + ddev->sid = sid;
> > > + TAILQ_INSERT_TAIL(&dom->devices, ddev, link);
> > > +
> > > + /* Should set context entry here?? */
> > > +}
> > > +
> > > +void
> > > +domain_remove_device(struct domain *dom, int sid)
> > > +{
> > > + struct domain_dev *ddev, *tmp;
> > > +
> > > + TAILQ_FOREACH_SAFE(ddev, &dom->devices, link, tmp) {
> > > + if (ddev->sid == sid) {
> > > + TAILQ_REMOVE(&dom->devices, ddev, link);
> > > + free(ddev, sizeof(*ddev), M_DEVBUF);
> > > + }
> > > + }
> > > +}
> > > +
> > > +/* Lookup domain by segment & source id (bus.device.function) */
> > > +struct domain *
> > > +domain_lookup(struct acpidmar_softc *sc, int segment, int sid)
> > > +{
> > > + struct iommu_softc *iommu;
> > > + struct domain_dev *ddev;
> > > + struct domain *dom;
> > > + int rc;
> > > +
> > > + if (sc == NULL) {
> > > + return NULL;
> > > + }
> > > +
> > > + /* Lookup IOMMU for this device */
> > > + TAILQ_FOREACH(iommu, &sc->sc_drhds, link) {
> > > + if (iommu->segment != segment)
> > > + continue;
> > > + /* Check for devscope match or catchall iommu */
> > > + rc = acpidmar_match_devscope(&iommu->devices, sc->sc_pc, sid);
> > > + if (rc != 0 || iommu->flags) {
> > > + break;
> > > + }
> > > + }
> > > + if (!iommu) {
> > > + printf("%s: no iommu found\n", dmar_bdf(sid));
> > > + return NULL;
> > > + }
> > > +
> > > + //acpidmar_intr(iommu);
> >
> > ?
> >
> > > +
> > > + /* Search domain devices */
> > > + TAILQ_FOREACH(dom, &iommu->domains, link) {
> > > + TAILQ_FOREACH(ddev, &dom->devices, link) {
> > > + /* XXX: match all functions? */
> > > + if (ddev->sid == sid) {
> > > + return dom;
> > > + }
> > > + }
> > > + }
> > > + if (iommu->ndoms <= 2) {
> > > + /* Running out of domains.. create catchall domain */
> > > + if (!iommu->unity) {
> > > + iommu->unity = domain_create(iommu, 1);
> > > + }
> > > + dom = iommu->unity;
> > > + } else {
> > > + dom = domain_create(iommu, --iommu->ndoms);
> > > + }
> > > + if (!dom) {
> > > + printf("no domain here\n");
> > > + return NULL;
> > > + }
> > > +
> > > + /* Add device to domain */
> > > + domain_add_device(dom, sid);
> > > +
> > > + return dom;
> > > +}
> > > +
> > > +/* Map Guest Pages into IOMMU */
> > > +void
> > > +_iommu_map(void *dom, vaddr_t va, bus_addr_t gpa, bus_size_t len)
> > > +{
> > > + bus_size_t i;
> > > + paddr_t hpa;
> > > +
> > > + if (dom == NULL) {
> > > + return;
> > > + }
> > > + dprintf("Mapping dma: %lx = %lx/%lx\n", va, gpa, len);
> > > + for (i = 0; i < len; i += PAGE_SIZE) {
> > > + hpa = 0;
> > > + pmap_extract(curproc->p_vmspace->vm_map.pmap, va, &hpa);
> > > + domain_map_page(dom, gpa, hpa, PTE_P | PTE_R | PTE_W);
> > > + gpa += PAGE_SIZE;
> > > + va += PAGE_SIZE;
> > > + }
> > > +}
> > > +
> > > +/* Find IOMMU for a given PCI device */
> > > +void
> > > +*_iommu_domain(int segment, int bus, int dev, int func, int *id)
> > > +{
> > > + struct domain *dom;
> > > +
> > > + dom = domain_lookup(acpidmar_sc, segment, mksid(bus, dev, func));
> > > + if (dom) {
> > > + *id = dom->did;
> > > + }
> > > + return dom;
> > > +}
> > > +
> > > +void
> > > +domain_map_device(struct domain *dom, int sid);
> > > +
> > > +void
> > > +domain_map_device(struct domain *dom, int sid)
> > > +{
> > > + struct iommu_softc *iommu;
> > > + struct context_entry *ctx;
> > > + paddr_t paddr;
> > > + int bus, devfn;
> > > + int tt, lvl;
> > > +
> > > + iommu = dom->iommu;
> > > +
> > > + bus = sid_bus(sid);
> > > + devfn = sid_devfn(sid);
> > > + /* AMD attach device */
> > > + if (iommu->dte) {
> > > + struct ivhd_dte *dte = &iommu->dte[sid];
> > > + if (!dte->dw0) {
> > > + /* Setup Device Table Entry: bus.devfn */
> > > + dprintf("@@@ PCI Attach: %.4x[%s] %.4x\n", sid,
> > > dmar_bdf(sid), dom->did);
> > > + dte_set_host_page_table_root_ptr(dte, dom->ptep);
> > > + dte_set_domain(dte, dom->did);
> > > + dte_set_mode(dte, 3); // Set 4 level PTE
> > > + dte_set_tv(dte);
> > > + dte_set_valid(dte);
> > > + ivhd_flush_devtab(iommu, dom->did);
> > > +#ifdef IOMMU_DEBUG
> > > + //ivhd_showreg(iommu);
> > > + ivhd_showdte(iommu);
> > > +#endif
> > > + }
> > > + //ivhd_poll_events(iommu);
> > > + return;
> > > + }
> > > +
> > > + /* Create Bus mapping */
> > > + if (!root_entry_is_valid(&iommu->root[bus])) {
> > > + iommu->ctx[bus] = iommu_alloc_page(iommu, &paddr);
> > > + iommu->root[bus].lo = paddr | ROOT_P;
> > > + iommu_flush_cache(iommu, &iommu->root[bus],
> > > + sizeof(struct root_entry));
> > > + dprintf("iommu%d: Allocate context for bus: %.2x pa:%.16llx
> > > va:%p\n",
> > > + iommu->id, bus, (uint64_t)paddr,
> > > + iommu->ctx[bus]);
> > > + }
> > > +
> > > + /* Create DevFn mapping */
> > > + ctx = iommu->ctx[bus] + devfn;
> > > + if (!context_entry_is_valid(ctx)) {
> > > + tt = CTX_T_MULTI;
> > > + lvl = VTD_AWTOLEVEL(iommu->agaw);
> > > +
> > > + /* Initialize context */
> > > + context_set_slpte(ctx, dom->ptep);
> > > + context_set_translation_type(ctx, tt);
> > > + context_set_domain_id(ctx, dom->did);
> > > + context_set_address_width(ctx, lvl);
> > > + context_set_present(ctx);
> > > +
> > > + /* Flush it */
> > > + iommu_flush_cache(iommu, ctx, sizeof(struct context_entry));
> > > + if ((iommu->cap & CAP_CM) || force_cm) {
> > > + iommu_flush_ctx(iommu, CTX_DEVICE, dom->did, sid, 0);
> > > + iommu_flush_tlb(iommu, IOTLB_GLOBAL, 0);
> > > + } else {
> > > + iommu_flush_write_buffer(iommu);
> > > + }
> > > + dprintf("iommu%d: %s set context ptep:%.16llx lvl:%d did:%.4x
> > > tt:%d\n",
> > > + iommu->id, dmar_bdf(sid), (uint64_t)dom->ptep, lvl,
> > > + dom->did, tt);
> > > + }
> > > +}
> > > +
> > > +struct domain *
> > > +acpidmar_pci_attach(struct acpidmar_softc *sc, int segment, int sid, int
> > > mapctx)
> > > +{
> > > + static struct domain *dom;
> > > +
> > > + dom = domain_lookup(sc, segment, sid);
> > > + if (!dom) {
> > > + printf("no domain: %s\n", dmar_bdf(sid));
> > > + return NULL;
> > > + }
> > > +
> > > + if (mapctx) {
> > > + domain_map_device(dom, sid);
> > > + }
> > > +
> > > + return dom;
> > > +}
> > > +
> > > +int ismap(int bus, int dev, int fun) {
> > > + return 1;
> > > +}
> >
> > Unused function.
> >
> > > +
> > > +void
> > > +acpidmar_pci_hook(pci_chipset_tag_t pc, struct pci_attach_args *pa)
> > > +{
> > > + int bus, dev, fun, sid;
> > > + struct domain *dom;
> > > + pcireg_t reg;
> > > +
> > > + if (!acpidmar_sc) {
> > > + /* No DMAR, ignore */
> > > + return;
> > > + }
> > > +
> > > + /* Add device to our list */
> > > + pci_decompose_tag(pc, pa->pa_tag, &bus, &dev, &fun);
> > > + sid = mksid(bus, dev, fun);
> > > + if (sid_flag[sid] & SID_INVALID)
> > > + return;
> > > +
> > > + reg = pci_conf_read(pc, pa->pa_tag, PCI_CLASS_REG);
> > > +#if 0
> > > + if (PCI_CLASS(reg) == PCI_CLASS_DISPLAY &&
> > > + PCI_SUBCLASS(reg) == PCI_SUBCLASS_DISPLAY_VGA) {
> > > + printf("dmar: %.4x:%.2x:%.2x.%x is VGA, ignoring\n",
> > > + pa->pa_domain, bus, dev, fun);
> > > + return;
> > > + }
> > > +#endif
> >
> > Remove? You seem to be handling this case below.
> >
> > > + /* Add device to domain */
> > > + dom = acpidmar_pci_attach(acpidmar_sc, pa->pa_domain, sid, 0);
> > > + if (dom == NULL)
> > > + return;
> > > +
> > > + if (PCI_CLASS(reg) == PCI_CLASS_DISPLAY &&
> > > + PCI_SUBCLASS(reg) == PCI_SUBCLASS_DISPLAY_VGA) {
> > > + dom->flag = DOM_NOMAP;
> > > + }
> > > + if (PCI_CLASS(reg) == PCI_CLASS_BRIDGE &&
> > > + PCI_SUBCLASS(reg) == PCI_SUBCLASS_BRIDGE_ISA) {
> > > + /* For ISA Bridges, map 0-16Mb as 1:1 */
> > > + printf("dmar: %.4x:%.2x:%.2x.%x mapping ISA\n",
> > > + pa->pa_domain, bus, dev, fun);
> > > + domain_map_pthru(dom, 0x00, 16*1024*1024);
> > > + }
> > > +
> > > + /* Change DMA tag */
> > > + pa->pa_dmat = &dom->dmat;
> > > +}
> > > +
> > > +/* Create list of device scope entries from ACPI table */
> > > +void
> > > +acpidmar_parse_devscope(union acpidmar_entry *de, int off, int segment,
> > > + struct devlist_head *devlist)
> > > +{
> > > + struct acpidmar_devscope *ds;
> > > + struct dmar_devlist *d;
> > > + int dplen, i;
> > > +
> > > + TAILQ_INIT(devlist);
> > > + while (off < de->length) {
> > > + ds = (struct acpidmar_devscope *)((unsigned char *)de + off);
> > > + off += ds->length;
> > > +
> > > + /* We only care about bridges and endpoints */
> > > + if (ds->type != DMAR_ENDPOINT && ds->type != DMAR_BRIDGE)
> > > + continue;
> > > +
> > > + dplen = ds->length - sizeof(*ds);
> > > + d = malloc(sizeof(*d) + dplen, M_DEVBUF, M_ZERO | M_WAITOK);
> > > + d->bus = ds->bus;
> > > + d->type = ds->type;
> > > + d->ndp = dplen / 2;
> > > + d->dp = (void *)&d[1];
> > > + memcpy(d->dp, &ds[1], dplen);
> > > + TAILQ_INSERT_TAIL(devlist, d, link);
> > > +
> > > + printf(" %8s %.4x:%.2x.%.2x.%x {",
> > > + ds->type == DMAR_BRIDGE ? "bridge" : "endpoint",
> > > + segment, ds->bus,
> > > + d->dp[0].device,
> > > + d->dp[0].function);
> > > +
> > > + for (i = 1; i < d->ndp; i++) {
> > > + printf(" %2x.%x ",
> > > + d->dp[i].device,
> > > + d->dp[i].function);
> > > + }
> > > + printf("}\n");
> >
> > More debug printf stuff?
> >
> > > + }
> > > +}
> > > +
> > > +/* DMA Remapping Hardware Unit */
>
>
> diff --git a/sys/arch/amd64/conf/GENERIC b/sys/arch/amd64/conf/GENERIC
> index ea43a2158..8ea728311 100644
> --- a/sys/arch/amd64/conf/GENERIC
> +++ b/sys/arch/amd64/conf/GENERIC
> @@ -45,6 +45,7 @@ acpibtn* at acpi?
> acpicpu* at acpi?
> acpicmos* at acpi?
> acpidock* at acpi?
> +acpidmar0 at acpi?
> acpiec* at acpi?
> acpipci* at acpi?
> acpiprt* at acpi?
> diff --git a/sys/arch/amd64/include/pci_machdep.h
> b/sys/arch/amd64/include/pci_machdep.h
> index bc295cc22..c725bdc73 100644
> --- a/sys/arch/amd64/include/pci_machdep.h
> +++ b/sys/arch/amd64/include/pci_machdep.h
> @@ -91,7 +91,8 @@ void
> *pci_intr_establish_cpu(pci_chipset_tag_t, pci_intr_handle_t,
> int, struct cpu_info *,
> int (*)(void *), void *, const char *);
> void pci_intr_disestablish(pci_chipset_tag_t, void *);
> -#define pci_probe_device_hook(c, a) (0)
> +int pci_probe_device_hook(pci_chipset_tag_t,
> + struct pci_attach_args *);
>
> void pci_dev_postattach(struct device *, struct
> pci_attach_args *);
>
> diff --git a/sys/arch/amd64/pci/pci_machdep.c
> b/sys/arch/amd64/pci/pci_machdep.c
> index cf4e835de..80d4f4d86 100644
> --- a/sys/arch/amd64/pci/pci_machdep.c
> +++ b/sys/arch/amd64/pci/pci_machdep.c
> @@ -89,6 +89,13 @@
> #include <machine/mpbiosvar.h>
> #endif
>
> +#include "acpi.h"
> +
> +#include "acpidmar.h"
> +#if NACPIDMAR > 0
> +#include <dev/acpi/acpidmar.h>
> +#endif
> +
> /*
> * Memory Mapped Configuration space access.
> *
> @@ -797,7 +804,15 @@ pci_init_extents(void)
> }
> }
>
> -#include "acpi.h"
> +int
> +pci_probe_device_hook(pci_chipset_tag_t pc, struct pci_attach_args *pa)
> +{
> +#if NACPIDMAR > 0
> + acpidmar_pci_hook(pc, pa);
> +#endif
> + return 0;
> +}
> +
> #if NACPI > 0
> void acpi_pci_match(struct device *, struct pci_attach_args *);
> pcireg_t acpi_pci_min_powerstate(pci_chipset_tag_t, pcitag_t);
> diff --git a/sys/dev/acpi/acpidmar.c b/sys/dev/acpi/acpidmar.c
> new file mode 100644
> index 000000000..0c21a92a2
> --- /dev/null
> +++ b/sys/dev/acpi/acpidmar.c
> @@ -0,0 +1,2913 @@
> +/*
> + * Copyright (c) 2015 Jordan Hargrave <jordan_hargr...@hotmail.com>
> + *
> + * Permission to use, copy, modify, and distribute this software for any
> + * purpose with or without fee is hereby granted, provided that the above
> + * copyright notice and this permission notice appear in all copies.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> + */
> +
> +#include <sys/param.h>
> +#include <sys/systm.h>
> +#include <sys/kernel.h>
> +#include <sys/device.h>
> +#include <sys/malloc.h>
> +#include <sys/queue.h>
> +#include <sys/types.h>
> +#include <sys/mbuf.h>
> +#include <sys/proc.h>
> +
> +#include <uvm/uvm_extern.h>
> +
> +#include <machine/apicvar.h>
> +#include <machine/biosvar.h>
> +#include <machine/cpuvar.h>
> +#include <machine/bus.h>
> +
> +#include <dev/acpi/acpireg.h>
> +#include <dev/acpi/acpivar.h>
> +#include <dev/acpi/acpidev.h>
> +#include <dev/acpi/amltypes.h>
> +#include <dev/acpi/dsdt.h>
> +
> +#include <uvm/uvm_extern.h>
> +
> +#include <machine/i8259.h>
> +#include <machine/i82093reg.h>
> +#include <machine/i82093var.h>
> +#include <machine/i82489reg.h>
> +#include <machine/i82489var.h>
> +
> +#include <machine/mpbiosvar.h>
> +
> +#include <dev/pci/pcireg.h>
> +#include <dev/pci/pcivar.h>
> +#include <dev/pci/pcidevs.h>
> +#include <dev/pci/ppbreg.h>
> +
> +#include "ioapic.h"
> +
> +#include "acpidmar.h"
> +#include "amd_iommu.h"
> +
> +/* We don't want IOMMU to remap MSI */
> +#define MSI_BASE_ADDRESS 0xFEE00000L
> +#define MSI_BASE_SIZE 0x00100000L
> +#define MAX_DEVFN 65536
> +
> +#ifdef IOMMU_DEBUG
> +int acpidmar_dbg_lvl = 0;
> +#define DPRINTF(lvl,x...) if (acpidmar_dbg_lvl >= lvl) { printf(x) }
> +#else
> +#define DPRINTF(lvl,x...)
> +#endif
> +
> +#ifdef DDB
> +int acpidmar_ddb = 0;
> +#endif
> +
> +int acpidmar_force_cm = 1;
> +
> +/* Page Table Entry per domain */
> +struct iommu_softc;
> +
> +static inline int
> +mksid(int b, int d, int f)
> +{
> + return (b << 8) + (d << 3) + f;
> +}
> +
> +static inline int
> +sid_devfn(int sid)
> +{
> + return sid & 0xff;
> +}
> +
> +static inline int
> +sid_bus(int sid)
> +{
> + return (sid >> 8) & 0xff;
> +}
> +
> +static inline int
> +sid_dev(int sid)
> +{
> + return (sid >> 3) & 0x1f;
> +}
> +
> +static inline int
> +sid_fun(int sid)
> +{
> + return (sid >> 0) & 0x7;
> +}
> +
> +/* Alias mapping */
> +#define SID_INVALID 0x80000000L
> +static uint32_t sid_flag[MAX_DEVFN];
> +
> +struct domain_dev {
> + int sid;
> + int sec;
> + int sub;
> + TAILQ_ENTRY(domain_dev) link;
> +};
> +
> +struct domain {
> + struct iommu_softc *iommu;
> + int did;
> + int gaw;
> + struct pte_entry *pte;
> + paddr_t ptep;
> + struct bus_dma_tag dmat;
> + int flag;
> +
> + struct mutex exlck;
> + char exname[32];
> + struct extent *iovamap;
> + TAILQ_HEAD(,domain_dev) devices;
> + TAILQ_ENTRY(domain) link;
> +};
> +
> +#define DOM_DEBUG 0x1
> +#define DOM_NOMAP 0x2
> +
> +struct dmar_devlist {
> + int type;
> + int bus;
> + int ndp;
> + struct acpidmar_devpath *dp;
> + TAILQ_ENTRY(dmar_devlist) link;
> +};
> +
> +TAILQ_HEAD(devlist_head, dmar_devlist);
> +
> +struct ivhd_devlist {
> + int start_id;
> + int end_id;
> + int cfg;
> + TAILQ_ENTRY(ivhd_devlist) link;
> +};
> +
> +struct rmrr_softc {
> + TAILQ_ENTRY(rmrr_softc) link;
> + struct devlist_head devices;
> + int segment;
> + uint64_t start;
> + uint64_t end;
> +};
> +
> +struct atsr_softc {
> + TAILQ_ENTRY(atsr_softc) link;
> + struct devlist_head devices;
> + int segment;
> + int flags;
> +};
> +
> +struct iommu_pic {
> + struct pic pic;
> + struct iommu_softc *iommu;
> +};
> +
> +#define IOMMU_FLAGS_CATCHALL 0x1
> +#define IOMMU_FLAGS_BAD 0x2
> +#define IOMMU_FLAGS_SUSPEND 0x4
> +
> +struct iommu_softc {
> + TAILQ_ENTRY(iommu_softc)link;
> + struct devlist_head devices;
> + int id;
> + int flags;
> + int segment;
> +
> + struct mutex reg_lock;
> +
> + bus_space_tag_t iot;
> + bus_space_handle_t ioh;
> +
> + uint64_t cap;
> + uint64_t ecap;
> + uint32_t gcmd;
> +
> + int mgaw;
> + int agaw;
> + int ndoms;
> +
> + struct root_entry *root;
> + struct context_entry *ctx[256];
> +
> + void *intr;
> + struct iommu_pic pic;
> + int fedata;
> + uint64_t feaddr;
> + uint64_t rtaddr;
> +
> + /* Queued Invalidation */
> + int qi_head;
> + int qi_tail;
> + paddr_t qip;
> + struct qi_entry *qi;
> +
> + struct domain *unity;
> + TAILQ_HEAD(,domain) domains;
> +
> + /* AMD iommu */
> + struct ivhd_dte *dte;
> + void *cmd_tbl;
> + void *evt_tbl;
> + paddr_t cmd_tblp;
> + paddr_t evt_tblp;
> +};
> +
> +static inline int iommu_bad(struct iommu_softc *sc)
> +{
> + return (sc->flags & IOMMU_FLAGS_BAD);
> +}
> +
> +static inline int iommu_enabled(struct iommu_softc *sc)
> +{
> + if (sc->dte) {
> + return 1;
> + }
> + return (sc->gcmd & GCMD_TE);
> +}
> +
> +struct acpidmar_softc {
> + struct device sc_dev;
> +
> + pci_chipset_tag_t sc_pc;
> + bus_space_tag_t sc_memt;
> + int sc_haw;
> + int sc_flags;
> + struct bus_dma_tag sc_dmat;
> +
> + struct ivhd_dte *sc_hwdte;
> + paddr_t sc_hwdtep;
> +
> + TAILQ_HEAD(,iommu_softc)sc_drhds;
> + TAILQ_HEAD(,rmrr_softc) sc_rmrrs;
> + TAILQ_HEAD(,atsr_softc) sc_atsrs;
> +};
> +
> +int acpidmar_activate(struct device *, int);
> +int acpidmar_match(struct device *, void *, void *);
> +void acpidmar_attach(struct device *, struct device *, void *);
> +struct domain *acpidmar_pci_attach(struct acpidmar_softc *, int, int, int);
> +
> +struct cfattach acpidmar_ca = {
> + sizeof(struct acpidmar_softc), acpidmar_match, acpidmar_attach, NULL,
> + acpidmar_activate
> +};
> +
> +struct cfdriver acpidmar_cd = {
> + NULL, "acpidmar", DV_DULL
> +};
> +
> +struct acpidmar_softc *acpidmar_sc;
> +int acpidmar_intr(void *);
> +int acpiivhd_intr(void *);
> +
> +#define DID_UNITY 0x1
> +
> +void _dumppte(struct pte_entry *, int, vaddr_t);
> +
> +struct domain *domain_create(struct iommu_softc *, int);
> +struct domain *domain_lookup(struct acpidmar_softc *, int, int);
> +
> +void domain_unload_map(struct domain *, bus_dmamap_t);
> +void domain_load_map(struct domain *, bus_dmamap_t, int, int, const char *);
> +
> +void (*domain_map_page)(struct domain *, vaddr_t, paddr_t, uint64_t);
> +void domain_map_page_amd(struct domain *, vaddr_t, paddr_t, uint64_t);
> +void domain_map_page_intel(struct domain *, vaddr_t, paddr_t, uint64_t);
> +void domain_map_pthru(struct domain *, paddr_t, paddr_t);
> +
> +void acpidmar_pci_hook(pci_chipset_tag_t, struct pci_attach_args *);
> +void acpidmar_parse_devscope(union acpidmar_entry *, int, int,
> + struct devlist_head *);
> +int acpidmar_match_devscope(struct devlist_head *, pci_chipset_tag_t, int);
> +
> +void acpidmar_init(struct acpidmar_softc *, struct acpi_dmar *);
> +void acpidmar_drhd(struct acpidmar_softc *, union acpidmar_entry *);
> +void acpidmar_rmrr(struct acpidmar_softc *, union acpidmar_entry *);
> +void acpidmar_atsr(struct acpidmar_softc *, union acpidmar_entry *);
> +void acpiivrs_init(struct acpidmar_softc *, struct acpi_ivrs *);
> +
> +void *acpidmar_intr_establish(void *, int, int (*)(void *), void *,
> + const char *);
> +
> +void iommu_write_4(struct iommu_softc *, int, uint32_t);
> +uint32_t iommu_read_4(struct iommu_softc *, int);
> +void iommu_write_8(struct iommu_softc *, int, uint64_t);
> +uint64_t iommu_read_8(struct iommu_softc *, int);
> +void iommu_showfault(struct iommu_softc *, int,
> + struct fault_entry *);
> +void iommu_showcfg(struct iommu_softc *, int);
> +
> +int iommu_init(struct acpidmar_softc *, struct iommu_softc *,
> + struct acpidmar_drhd *);
> +int iommu_enable_translation(struct iommu_softc *, int);
> +void iommu_enable_qi(struct iommu_softc *, int);
> +void iommu_flush_cache(struct iommu_softc *, void *, size_t);
> +void *iommu_alloc_page(struct iommu_softc *, paddr_t *);
> +void iommu_flush_write_buffer(struct iommu_softc *);
> +void iommu_issue_qi(struct iommu_softc *, struct qi_entry *);
> +
> +void iommu_flush_ctx(struct iommu_softc *, int, int, int, int);
> +void iommu_flush_ctx_qi(struct iommu_softc *, int, int, int, int);
> +void iommu_flush_tlb(struct iommu_softc *, int, int);
> +void iommu_flush_tlb_qi(struct iommu_softc *, int, int);
> +
> +void iommu_set_rtaddr(struct iommu_softc *, paddr_t);
> +
> +void *iommu_alloc_hwdte(struct acpidmar_softc *, size_t, paddr_t *);
> +
> +const char *dmar_bdf(int);
> +
> +const char *
> +dmar_bdf(int sid)
> +{
> + static char bdf[32];
> +
> + snprintf(bdf, sizeof(bdf), "%.4x:%.2x:%.2x.%x", 0,
> + sid_bus(sid), sid_dev(sid), sid_fun(sid));
> +
> + return (bdf);
> +}
> +
> +/* busdma */
> +static int dmar_dmamap_create(bus_dma_tag_t, bus_size_t, int, bus_size_t,
> + bus_size_t, int, bus_dmamap_t *);
> +static void dmar_dmamap_destroy(bus_dma_tag_t, bus_dmamap_t);
> +static int dmar_dmamap_load(bus_dma_tag_t, bus_dmamap_t, void *, bus_size_t,
> + struct proc *, int);
> +static int dmar_dmamap_load_mbuf(bus_dma_tag_t, bus_dmamap_t, struct mbuf *,
> + int);
> +static int dmar_dmamap_load_uio(bus_dma_tag_t, bus_dmamap_t, struct uio *,
> int);
> +static int dmar_dmamap_load_raw(bus_dma_tag_t, bus_dmamap_t,
> + bus_dma_segment_t *, int, bus_size_t, int);
> +static void dmar_dmamap_unload(bus_dma_tag_t, bus_dmamap_t);
> +static void dmar_dmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
> + bus_size_t, int);
> +static int dmar_dmamem_alloc(bus_dma_tag_t, bus_size_t, bus_size_t,
> bus_size_t,
> + bus_dma_segment_t *, int, int *, int);
> +static void dmar_dmamem_free(bus_dma_tag_t, bus_dma_segment_t *, int);
> +static int dmar_dmamem_map(bus_dma_tag_t, bus_dma_segment_t *, int, size_t,
> + caddr_t *, int);
> +static void dmar_dmamem_unmap(bus_dma_tag_t, caddr_t, size_t);
> +static paddr_t dmar_dmamem_mmap(bus_dma_tag_t, bus_dma_segment_t *,
> int, off_t,
> + int, int);
> +
> +static void dmar_dumpseg(bus_dma_tag_t, int, bus_dma_segment_t *, const char
> *);
> +const char *dom_bdf(struct domain *);
> +void domain_map_check(struct domain *);
> +
> +struct pte_entry *pte_lvl(struct iommu_softc *, struct pte_entry *, vaddr_t,
> int, uint64_t);
> +int ivhd_poll_events(struct iommu_softc *);
> +void ivhd_showreg(struct iommu_softc *);
> +void ivhd_showdte(struct iommu_softc *);
> +void ivhd_showcmd(struct iommu_softc *);
> +
> +static inline int
> +debugme(struct domain *dom)
> +{
> + return 0;
> + return (dom->flag & DOM_DEBUG);
> +}
> +
> +void
> +domain_map_check(struct domain *dom)
> +{
> + struct iommu_softc *iommu;
> + struct domain_dev *dd;
> + struct context_entry *ctx;
> + int v;
> +
> + iommu = dom->iommu;
> + TAILQ_FOREACH(dd, &dom->devices, link) {
> + acpidmar_pci_attach(acpidmar_sc, iommu->segment, dd->sid, 1);
> +
> + if (iommu->dte)
> + continue;
> +
> + /* Check if this is the first time we are mapped */
> + ctx = &iommu->ctx[sid_bus(dd->sid)][sid_devfn(dd->sid)];
> + v = context_user(ctx);
> + if (v != 0xA) {
> + printf(" map: %.4x:%.2x:%.2x.%x iommu:%d did:%.4x\n",
> + iommu->segment,
> + sid_bus(dd->sid),
> + sid_dev(dd->sid),
> + sid_fun(dd->sid),
> + iommu->id,
> + dom->did);
> + context_set_user(ctx, 0xA);
> + }
> + }
> +}
> +
> +/* Map a single page as passthrough - used for DRM */
> +void
> +dmar_ptmap(bus_dma_tag_t tag, bus_addr_t addr)
> +{
> + struct domain *dom = tag->_cookie;
> +
> + if (!acpidmar_sc)
> + return;
> + domain_map_check(dom);
> + domain_map_page(dom, addr, addr, PTE_P | PTE_R | PTE_W);
> +}
> +
> +/* Map a range of pages 1:1 */
> +void
> +domain_map_pthru(struct domain *dom, paddr_t start, paddr_t end)
> +{
> + domain_map_check(dom);
> + while (start < end) {
> + domain_map_page(dom, start, start, PTE_P | PTE_R | PTE_W);
> + start += VTD_PAGE_SIZE;
> + }
> +}
> +
> +/* Map a single paddr to IOMMU paddr */
> +void
> +domain_map_page_intel(struct domain *dom, vaddr_t va, paddr_t pa, uint64_t
> flags)
> +{
> + paddr_t paddr;
> + struct pte_entry *pte, *npte;
> + int lvl, idx;
> + struct iommu_softc *iommu;
> +
> + iommu = dom->iommu;
> + /* Insert physical address into virtual address map
> + * XXX: could we use private pmap here?
> + * essentially doing a pmap_enter(map, va, pa, prot);
> + */
> +
> + /* Only handle 4k pages for now */
> + npte = dom->pte;
> + for (lvl = iommu->agaw - VTD_STRIDE_SIZE; lvl>= VTD_LEVEL0;
> + lvl -= VTD_STRIDE_SIZE) {
> + idx = (va >> lvl) & VTD_STRIDE_MASK;
> + pte = &npte[idx];
> + if (lvl == VTD_LEVEL0) {
> + /* Level 1: Page Table - add physical address */
> + pte->val = pa | flags;
> + iommu_flush_cache(iommu, pte, sizeof(*pte));
> + break;
> + } else if (!(pte->val & PTE_P)) {
> + /* Level N: Point to lower level table */
> + iommu_alloc_page(iommu, &paddr);
> + pte->val = paddr | PTE_P | PTE_R | PTE_W;
> + iommu_flush_cache(iommu, pte, sizeof(*pte));
> + }
> + npte = (void *)PMAP_DIRECT_MAP((pte->val & VTD_PTE_MASK));
> + }
> +}
> +
> +/* Map a single paddr to IOMMU paddr: AMD
> + * physical address breakdown into levels:
> + * xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx
> + * 5.55555555.44444444.43333333,33222222.22211111.1111----.--------
> + * mode:
> + * 000 = none shift
> + * 001 = 1 [21].12
> + * 010 = 2 [30].21
> + * 011 = 3 [39].30
> + * 100 = 4 [48].39
> + * 101 = 5 [57]
> + * 110 = 6
> + * 111 = reserved
> + */
> +struct pte_entry *
> +pte_lvl(struct iommu_softc *iommu, struct pte_entry *pte, vaddr_t va,
> + int shift, uint64_t flags)
> +{
> + paddr_t paddr;
> + int idx;
> +
> + idx = (va >> shift) & VTD_STRIDE_MASK;
> + if (!(pte[idx].val & PTE_P)) {
> + /* Page Table entry is not present... create a new page entry */
> + iommu_alloc_page(iommu, &paddr);
> + pte[idx].val = paddr | flags;
> + iommu_flush_cache(iommu, &pte[idx], sizeof(pte[idx]));
> + }
> + return (void *)PMAP_DIRECT_MAP((pte[idx].val & PTE_PADDR_MASK));
> +}
> +
> +void
> +domain_map_page_amd(struct domain *dom, vaddr_t va, paddr_t pa, uint64_t
> flags)
> +{
> + struct pte_entry *pte;
> + struct iommu_softc *iommu;
> + int idx;
> +
> + iommu = dom->iommu;
> + /* Insert physical address into virtual address map
> + * XXX: could we use private pmap here?
> + * essentially doing a pmap_enter(map, va, pa, prot);
> + */
> +
> + /* Always assume AMD levels=4 */
> + /* 39 30 21 12 */
> + /* ---------|---------|---------|---------|------------ */
> + pte = dom->pte;
> + pte = pte_lvl(iommu, pte, va, 30, PTE_NXTLVL(2) | PTE_IR | PTE_IW |
> PTE_P);
> + pte = pte_lvl(iommu, pte, va, 21, PTE_NXTLVL(1) | PTE_IR | PTE_IW |
> PTE_P);
> + if (flags)
> + flags = PTE_P | PTE_R | PTE_W | PTE_IW | PTE_IR | PTE_NXTLVL(0);
> +
> + /* Level 1: Page Table - add physical address */
> + idx = (va >> 12) & 0x1FF;
> + pte[idx].val = pa | flags;
> +
> + iommu_flush_cache(iommu, pte, sizeof(*pte));
> +}
> +
> +static void
> +dmar_dumpseg(bus_dma_tag_t tag, int nseg, bus_dma_segment_t *segs,
> + const char *lbl)
> +{
> + struct domain *dom = tag->_cookie;
> + int i;
> +
> + return;
> + if (!debugme(dom))
> + return;
> + printf("%s: %s\n", lbl, dom_bdf(dom));
> + for (i = 0; i < nseg; i++) {
> + printf(" %.16llx %.8x\n",
> + (uint64_t)segs[i].ds_addr,
> + (uint32_t)segs[i].ds_len);
> + }
> +}
> +
> +/* Unload mapping */
> +void
> +domain_unload_map(struct domain *dom, bus_dmamap_t dmam)
> +{
> + bus_dma_segment_t *seg;
> + paddr_t base, end, idx;
> + psize_t alen;
> + int i;
> +
> + if (iommu_bad(dom->iommu)) {
> + printf("unload map no iommu\n");
> + return;
> + }
> +
> + for (i = 0; i < dmam->dm_nsegs; i++) {
> + seg = &dmam->dm_segs[i];
> +
> + base = trunc_page(seg->ds_addr);
> + end = roundup(seg->ds_addr + seg->ds_len, VTD_PAGE_SIZE);
> + alen = end - base;
> +
> + if (debugme(dom)) {
> + printf(" va:%.16llx len:%x\n",
> + (uint64_t)base, (uint32_t)alen);
> + }
> +
> + /* Clear PTE */
> + for (idx = 0; idx < alen; idx += VTD_PAGE_SIZE)
> + domain_map_page(dom, base + idx, 0, 0);
> +
> + if (dom->flag & DOM_NOMAP) {
> + printf("%s: nomap %.16llx\n", dom_bdf(dom),
> (uint64_t)base);
> + continue;
> + }
> +
> + mtx_enter(&dom->exlck);
> + if (extent_free(dom->iovamap, base, alen, EX_NOWAIT)) {
> + panic("domain_unload_map: extent_free");
> + }
> + mtx_leave(&dom->exlck);
> + }
> +}
> +
> +/* map.segs[x].ds_addr is modified to IOMMU virtual PA */
> +void
> +domain_load_map(struct domain *dom, bus_dmamap_t map, int flags, int
> pteflag, const char *fn)
> +{
> + bus_dma_segment_t *seg;
> + struct iommu_softc *iommu;
> + paddr_t base, end, idx;
> + psize_t alen;
> + u_long res;
> + int i;
> +
> + iommu = dom->iommu;
> + if (!iommu_enabled(iommu)) {
> + /* Lazy enable translation when required */
> + if (iommu_enable_translation(iommu, 1)) {
> + return;
> + }
> + }
> + domain_map_check(dom);
> + for (i = 0; i < map->dm_nsegs; i++) {
> + seg = &map->dm_segs[i];
> +
> + base = trunc_page(seg->ds_addr);
> + end = roundup(seg->ds_addr + seg->ds_len, VTD_PAGE_SIZE);
> + alen = end - base;
> + res = base;
> +
> + if (dom->flag & DOM_NOMAP) {
> + goto nomap;
> + }
> +
> + /* Allocate DMA Virtual Address */
> + mtx_enter(&dom->exlck);
> + if (extent_alloc(dom->iovamap, alen, VTD_PAGE_SIZE, 0,
> + map->_dm_boundary, EX_NOWAIT, &res)) {
> + panic("domain_load_map: extent_alloc");
> + }
> + if (res == -1) {
> + panic("got -1 address\n");
> + }
> + mtx_leave(&dom->exlck);
> +
> + /* Reassign DMA address */
> + seg->ds_addr = res | (seg->ds_addr & VTD_PAGE_MASK);
> +nomap:
> + if (debugme(dom)) {
> + printf(" LOADMAP: %.16llx %x => %.16llx\n",
> + (uint64_t)seg->ds_addr, (uint32_t)seg->ds_len,
> + (uint64_t)res);
> + }
> + for (idx = 0; idx < alen; idx += VTD_PAGE_SIZE) {
> + domain_map_page(dom, res + idx, base + idx,
> + PTE_P | pteflag);
> + }
> + }
> + if ((iommu->cap & CAP_CM) || acpidmar_force_cm) {
> + iommu_flush_tlb(iommu, IOTLB_DOMAIN, dom->did);
> + } else {
> + iommu_flush_write_buffer(iommu);
> + }
> +}
> +
> +const char *
> +dom_bdf(struct domain *dom)
> +{
> + struct domain_dev *dd;
> + static char mmm[48];
> +
> + dd = TAILQ_FIRST(&dom->devices);
> + snprintf(mmm, sizeof(mmm), "%s iommu:%d did:%.4x%s",
> + dmar_bdf(dd->sid), dom->iommu->id, dom->did,
> + dom->did == DID_UNITY ? " [unity]" : "");
> + return (mmm);
> +}
> +
> +/* Bus DMA Map functions */
> +static int
> +dmar_dmamap_create(bus_dma_tag_t tag, bus_size_t size, int nsegments,
> + bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
> +{
> + int rc;
> +
> + rc = _bus_dmamap_create(tag, size, nsegments, maxsegsz, boundary,
> + flags, dmamp);
> + if (!rc) {
> + dmar_dumpseg(tag, (*dmamp)->dm_nsegs, (*dmamp)->dm_segs,
> + __FUNCTION__);
> + }
> + return (rc);
> +}
> +
> +static void
> +dmar_dmamap_destroy(bus_dma_tag_t tag, bus_dmamap_t dmam)
> +{
> + dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs, __FUNCTION__);
> + _bus_dmamap_destroy(tag, dmam);
> +}
> +
> +static int
> +dmar_dmamap_load(bus_dma_tag_t tag, bus_dmamap_t dmam, void *buf,
> + bus_size_t buflen, struct proc *p, int flags)
> +{
> + struct domain *dom = tag->_cookie;
> + int rc;
> +
> + rc = _bus_dmamap_load(tag, dmam, buf, buflen, p, flags);
> + if (!rc) {
> + dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
> + __FUNCTION__);
> + domain_load_map(dom, dmam, flags, PTE_R|PTE_W, __FUNCTION__);
> + dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
> + __FUNCTION__);
> + }
> + return (rc);
> +}
> +
> +static int
> +dmar_dmamap_load_mbuf(bus_dma_tag_t tag, bus_dmamap_t dmam, struct mbuf
> *chain,
> + int flags)
> +{
> + struct domain *dom = tag->_cookie;
> + int rc;
> +
> + rc = _bus_dmamap_load_mbuf(tag, dmam, chain, flags);
> + if (!rc) {
> + dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
> + __FUNCTION__);
> + domain_load_map(dom, dmam, flags, PTE_R|PTE_W,__FUNCTION__);
> + dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
> + __FUNCTION__);
> + }
> + return (rc);
> +}
> +
> +static int
> +dmar_dmamap_load_uio(bus_dma_tag_t tag, bus_dmamap_t dmam, struct uio *uio,
> + int flags)
> +{
> + struct domain *dom = tag->_cookie;
> + int rc;
> +
> + rc = _bus_dmamap_load_uio(tag, dmam, uio, flags);
> + if (!rc) {
> + dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
> + __FUNCTION__);
> + domain_load_map(dom, dmam, flags, PTE_R|PTE_W, __FUNCTION__);
> + dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
> + __FUNCTION__);
> + }
> + return (rc);
> +}
> +
> +static int
> +dmar_dmamap_load_raw(bus_dma_tag_t tag, bus_dmamap_t dmam,
> + bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
> +{
> + struct domain *dom = tag->_cookie;
> + int rc;
> +
> + rc = _bus_dmamap_load_raw(tag, dmam, segs, nsegs, size, flags);
> + if (!rc) {
> + dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
> + __FUNCTION__);
> + domain_load_map(dom, dmam, flags, PTE_R|PTE_W, __FUNCTION__);
> + dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
> + __FUNCTION__);
> + }
> + return (rc);
> +}
> +
> +static void
> +dmar_dmamap_unload(bus_dma_tag_t tag, bus_dmamap_t dmam)
> +{
> + struct domain *dom = tag->_cookie;
> +
> + dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs, __FUNCTION__);
> + domain_unload_map(dom, dmam);
> + _bus_dmamap_unload(tag, dmam);
> +}
> +
> +static void
> +dmar_dmamap_sync(bus_dma_tag_t tag, bus_dmamap_t dmam, bus_addr_t offset,
> + bus_size_t len, int ops)
> +{
> +#if 0
> + struct domain *dom = tag->_cookie;
> + int flag;
> +
> + flag = PTE_P;
> + if (ops == BUS_DMASYNC_PREREAD) {
> + /* make readable */
> + flag |= PTE_R;
> + }
> + else if (ops == BUS_DMASYNC_PREWRITE) {
> + /* make writeable */
> + flag |= PTE_W;
> + }
> + dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs, __FUNCTION__);
> +#endif
> + _bus_dmamap_sync(tag, dmam, offset, len, ops);
> +}
> +
> +static int
> +dmar_dmamem_alloc(bus_dma_tag_t tag, bus_size_t size, bus_size_t alignment,
> + bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
> + int flags)
> +{
> + int rc;
> +
> + rc = _bus_dmamem_alloc(tag, size, alignment, boundary, segs, nsegs,
> + rsegs, flags);
> + if (!rc) {
> + dmar_dumpseg(tag, *rsegs, segs, __FUNCTION__);
> + }
> + return (rc);
> +}
> +
> +static void
> +dmar_dmamem_free(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs)
> +{
> + dmar_dumpseg(tag, nsegs, segs, __FUNCTION__);
> + _bus_dmamem_free(tag, segs, nsegs);
> +}
> +
> +static int
> +dmar_dmamem_map(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs,
> + size_t size, caddr_t *kvap, int flags)
> +{
> + dmar_dumpseg(tag, nsegs, segs, __FUNCTION__);
> + return (_bus_dmamem_map(tag, segs, nsegs, size, kvap, flags));
> +}
> +
> +static void
> +dmar_dmamem_unmap(bus_dma_tag_t tag, caddr_t kva, size_t size)
> +{
> + struct domain *dom = tag->_cookie;
> +
> + if (debugme(dom)) {
> + printf("dmamap_unmap: %s\n", dom_bdf(dom));
> + }
> + _bus_dmamem_unmap(tag, kva, size);
> +}
> +
> +static paddr_t
> +dmar_dmamem_mmap(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs,
> + off_t off, int prot, int flags)
> +{
> + dmar_dumpseg(tag, nsegs, segs, __FUNCTION__);
> + return (_bus_dmamem_mmap(tag, segs, nsegs, off, prot, flags));
> +}
> +
> +/*===================================
> + * IOMMU code
> + *===================================*/
> +
> +/* Intel: Set Context Root Address */
> +void
> +iommu_set_rtaddr(struct iommu_softc *iommu, paddr_t paddr)
> +{
> + int i, sts;
> +
> + mtx_enter(&iommu->reg_lock);
> + iommu_write_8(iommu, DMAR_RTADDR_REG, paddr);
> + iommu_write_4(iommu, DMAR_GCMD_REG, iommu->gcmd | GCMD_SRTP);
> + for (i = 0; i < 5; i++) {
> + sts = iommu_read_4(iommu, DMAR_GSTS_REG);
> + if (sts & GSTS_RTPS)
> + break;
> + }
> + mtx_leave(&iommu->reg_lock);
> +
> + if (i == 5) {
> + printf("set_rtaddr fails\n");
> + }
> +}
> +
> +/* Allocate contiguous memory (1Mb) for the Device Table Entries */
> +void *
> +iommu_alloc_hwdte(struct acpidmar_softc *sc, size_t size, paddr_t *paddr)
> +{
> + caddr_t vaddr;
> + bus_dmamap_t map;
> + bus_dma_segment_t seg;
> + bus_dma_tag_t dmat;
> + int rc, nsegs;
> +
> + rc = _bus_dmamap_create(dmat, size, 1, size, 0,
> + BUS_DMA_NOWAIT, &map);
> + if (rc != 0) {
> + printf("hwdte_create fails\n");
> + return NULL;
> + }
> + rc = _bus_dmamem_alloc(dmat, size, 4, 0, &seg, 1,
> + &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO);
> + if (rc != 0) {
> + printf("hwdte alloc fails\n");
> + return NULL;
> + }
> + rc = _bus_dmamem_map(dmat, &seg, 1, size, &vaddr,
> + BUS_DMA_NOWAIT | BUS_DMA_COHERENT);
> + if (rc != 0) {
> + printf("hwdte map fails\n");
> + return NULL;
> + }
> + rc = _bus_dmamap_load_raw(dmat, map, &seg, 1, size, BUS_DMA_NOWAIT);
> + if (rc != 0) {
> + printf("hwdte load raw fails\n");
> + return NULL;
> + }
> + *paddr = map->dm_segs[0].ds_addr;
> + return vaddr;
> +}
> +
> +/* COMMON: Allocate a new memory page */
> +void *
> +iommu_alloc_page(struct iommu_softc *iommu, paddr_t *paddr)
> +{
> + void *va;
> +
> + *paddr = 0;
> + va = km_alloc(VTD_PAGE_SIZE, &kv_page, &kp_zero, &kd_nowait);
> + if (va == NULL) {
> + panic("can't allocate page\n");
> + }
> + pmap_extract(pmap_kernel(), (vaddr_t)va, paddr);
> + return (va);
> +}
> +
> +
> +/* Intel: Issue command via queued invalidation */
> +void
> +iommu_issue_qi(struct iommu_softc *iommu, struct qi_entry *qi)
> +{
> +#if 0
> + struct qi_entry *pi, *pw;
> +
> + idx = iommu->qi_head;
> + pi = &iommu->qi[idx];
> + pw = &iommu->qi[(idx+1) % MAXQ];
> + iommu->qi_head = (idx+2) % MAXQ;
> +
> + memcpy(pw, &qi, sizeof(qi));
> + issue command;
> + while (pw->xxx)
> + ;
> +#endif
> +}
> +
> +/* Intel: Flush TLB entries, Queued Invalidation mode */
> +void
> +iommu_flush_tlb_qi(struct iommu_softc *iommu, int mode, int did)
> +{
> + struct qi_entry qi;
> +
> + /* Use queued invalidation */
> + qi.hi = 0;
> + switch (mode) {
> + case IOTLB_GLOBAL:
> + qi.lo = QI_IOTLB | QI_IOTLB_IG_GLOBAL;
> + break;
> + case IOTLB_DOMAIN:
> + qi.lo = QI_IOTLB | QI_IOTLB_IG_DOMAIN |
> + QI_IOTLB_DID(did);
> + break;
> + case IOTLB_PAGE:
> + qi.lo = QI_IOTLB | QI_IOTLB_IG_PAGE | QI_IOTLB_DID(did);
> + qi.hi = 0;
> + break;
> + }
> + if (iommu->cap & CAP_DRD)
> + qi.lo |= QI_IOTLB_DR;
> + if (iommu->cap & CAP_DWD)
> + qi.lo |= QI_IOTLB_DW;
> + iommu_issue_qi(iommu, &qi);
> +}
> +
> +/* Intel: Flush Context entries, Queued Invalidation mode */
> +void
> +iommu_flush_ctx_qi(struct iommu_softc *iommu, int mode, int did,
> + int sid, int fm)
> +{
> + struct qi_entry qi;
> +
> + /* Use queued invalidation */
> + qi.hi = 0;
> + switch (mode) {
> + case CTX_GLOBAL:
> + qi.lo = QI_CTX | QI_CTX_IG_GLOBAL;
> + break;
> + case CTX_DOMAIN:
> + qi.lo = QI_CTX | QI_CTX_IG_DOMAIN | QI_CTX_DID(did);
> + break;
> + case CTX_DEVICE:
> + qi.lo = QI_CTX | QI_CTX_IG_DEVICE | QI_CTX_DID(did) |
> + QI_CTX_SID(sid) | QI_CTX_FM(fm);
> + break;
> + }
> + iommu_issue_qi(iommu, &qi);
> +}
> +
> +/* Intel: Flush write buffers */
> +void
> +iommu_flush_write_buffer(struct iommu_softc *iommu)
> +{
> + int i, sts;
> +
> + if (iommu->dte)
> + return;
> + if (!(iommu->cap & CAP_RWBF))
> + return;
> + DPRINTF(1,"writebuf\n");
> + iommu_write_4(iommu, DMAR_GCMD_REG, iommu->gcmd | GCMD_WBF);
> + for (i = 0; i < 5; i++) {
> + sts = iommu_read_4(iommu, DMAR_GSTS_REG);
> + if (sts & GSTS_WBFS)
> + break;
> + delay(10000);
> + }
> + if (i == 5) {
> + printf("write buffer flush fails\n");
> + }
> +}
> +
> +void
> +iommu_flush_cache(struct iommu_softc *iommu, void *addr, size_t size)
> +{
> + if (iommu->dte) {
> + pmap_flush_cache((vaddr_t)addr, size);
> + return;
> + }
> + if (!(iommu->ecap & ECAP_C))
> + pmap_flush_cache((vaddr_t)addr, size);
> +}
> +
> +/*
> + * Intel: Flush IOMMU TLB Entries
> + * Flushing can occur globally, per domain or per page
> + */
> +void
> +iommu_flush_tlb(struct iommu_softc *iommu, int mode, int did)
> +{
> + int n;
> + uint64_t val;
> +
> + /* Call AMD */
> + if (iommu->dte) {
> + ivhd_invalidate_domain(iommu, did);
> + return;
> + }
> + val = IOTLB_IVT;
> + switch (mode) {
> + case IOTLB_GLOBAL:
> + val |= IIG_GLOBAL;
> + break;
> + case IOTLB_DOMAIN:
> + val |= IIG_DOMAIN | IOTLB_DID(did);
> + break;
> + case IOTLB_PAGE:
> + val |= IIG_PAGE | IOTLB_DID(did);
> + break;
> + }
> +
> + /* Check for Read/Write Drain */
> + if (iommu->cap & CAP_DRD)
> + val |= IOTLB_DR;
> + if (iommu->cap & CAP_DWD)
> + val |= IOTLB_DW;
> +
> + mtx_enter(&iommu->reg_lock);
> +
> + iommu_write_8(iommu, DMAR_IOTLB_REG(iommu), val);
> + n = 0;
> + do {
> + val = iommu_read_8(iommu, DMAR_IOTLB_REG(iommu));
> + } while (n++ < 5 && val & IOTLB_IVT);
> +
> + mtx_leave(&iommu->reg_lock);
> +}
> +
> +/* Intel: Flush IOMMU settings
> + * Flushes can occur globally, per domain, or per device
> + */
> +void
> +iommu_flush_ctx(struct iommu_softc *iommu, int mode, int did, int sid, int
> fm)
> +{
> + uint64_t val;
> + int n;
> +
> + if (iommu->dte)
> + return;
> + val = CCMD_ICC;
> + switch (mode) {
> + case CTX_GLOBAL:
> + val |= CIG_GLOBAL;
> + break;
> + case CTX_DOMAIN:
> + val |= CIG_DOMAIN | CCMD_DID(did);
> + break;
> + case CTX_DEVICE:
> + val |= CIG_DEVICE | CCMD_DID(did) |
> + CCMD_SID(sid) | CCMD_FM(fm);
> + break;
> + }
> +
> + mtx_enter(&iommu->reg_lock);
> +
> + n = 0;
> + iommu_write_8(iommu, DMAR_CCMD_REG, val);
> + do {
> + val = iommu_read_8(iommu, DMAR_CCMD_REG);
> + } while (n++ < 5 && val & CCMD_ICC);
> +
> + mtx_leave(&iommu->reg_lock);
> +}
> +
> +/* Intel: Enable Queued Invalidation */
> +void
> +iommu_enable_qi(struct iommu_softc *iommu, int enable)
> +{
> + int n = 0;
> + int sts;
> +
> + if (!(iommu->ecap & ECAP_QI))
> + return;
> +
> + if (enable) {
> + iommu->gcmd |= GCMD_QIE;
> +
> + mtx_enter(&iommu->reg_lock);
> +
> + iommu_write_4(iommu, DMAR_GCMD_REG, iommu->gcmd);
> + do {
> + sts = iommu_read_4(iommu, DMAR_GSTS_REG);
> + } while (n++ < 5 && !(sts & GSTS_QIES));
> +
> + mtx_leave(&iommu->reg_lock);
> +
> + DPRINTF(1,"set.qie: %d\n", n);
> + } else {
> + iommu->gcmd &= ~GCMD_QIE;
> +
> + mtx_enter(&iommu->reg_lock);
> +
> + iommu_write_4(iommu, DMAR_GCMD_REG, iommu->gcmd);
> + do {
> + sts = iommu_read_4(iommu, DMAR_GSTS_REG);
> + } while (n++ < 5 && sts & GSTS_QIES);
> +
> + mtx_leave(&iommu->reg_lock);
> +
> + DPRINTF(1,"clr.qie: %d\n", n);
> + }
> +}
> +
> +/* Intel: Enable IOMMU translation */
> +int
> +iommu_enable_translation(struct iommu_softc *iommu, int enable)
> +{
> + uint32_t sts;
> + uint64_t reg;
> + int n = 0;
> +
> + if (iommu->dte)
> + return (0);
> + reg = 0;
> + if (enable) {
> + DPRINTF(0,"enable iommu %d\n", iommu->id);
> + iommu_showcfg(iommu, -1);
> +
> + iommu->gcmd |= GCMD_TE;
> +
> + /* Enable translation */
> + printf(" pre tes: ");
> +
> + mtx_enter(&iommu->reg_lock);
> + iommu_write_4(iommu, DMAR_GCMD_REG, iommu->gcmd);
> + printf("xxx");
> + do {
> + printf("yyy");
> + sts = iommu_read_4(iommu, DMAR_GSTS_REG);
> + delay(n * 10000);
> + } while (n++ < 5 && !(sts & GSTS_TES));
> + mtx_leave(&iommu->reg_lock);
> +
> + printf(" set.tes: %d\n", n);
> +
> + if (n >= 5) {
> + printf("error.. unable to initialize iommu %d\n",
> + iommu->id);
> + iommu->flags |= IOMMU_FLAGS_BAD;
> +
> + /* Disable IOMMU */
> + iommu->gcmd &= ~GCMD_TE;
> + mtx_enter(&iommu->reg_lock);
> + iommu_write_4(iommu, DMAR_GCMD_REG, iommu->gcmd);
> + mtx_leave(&iommu->reg_lock);
> +
> + return (1);
> + }
> +
> + iommu_flush_ctx(iommu, CTX_GLOBAL, 0, 0, 0);
> + iommu_flush_tlb(iommu, IOTLB_GLOBAL, 0);
> + } else {
> + iommu->gcmd &= ~GCMD_TE;
> +
> + mtx_enter(&iommu->reg_lock);
> +
> + iommu_write_4(iommu, DMAR_GCMD_REG, iommu->gcmd);
> + do {
> + sts = iommu_read_4(iommu, DMAR_GSTS_REG);
> + } while (n++ < 5 && sts & GSTS_TES);
> + mtx_leave(&iommu->reg_lock);
> +
> + printf(" clr.tes: %d\n", n);
> + }
> +
> + return (0);
> +}
> +
> +/* Intel: Initialize IOMMU */
> +int
> +iommu_init(struct acpidmar_softc *sc, struct iommu_softc *iommu,
> + struct acpidmar_drhd *dh)
> +{
> + static int niommu;
> + int len = VTD_PAGE_SIZE;
> + int i, gaw;
> + uint32_t sts;
> + paddr_t paddr;
> +
> + if (_bus_space_map(sc->sc_memt, dh->address, len, 0, &iommu->ioh) != 0)
> {
> + return (-1);
> + }
> +
> + TAILQ_INIT(&iommu->domains);
> + iommu->id = ++niommu;
> + iommu->flags = dh->flags;
> + iommu->segment = dh->segment;
> + iommu->iot = sc->sc_memt;
> +
> + iommu->cap = iommu_read_8(iommu, DMAR_CAP_REG);
> + iommu->ecap = iommu_read_8(iommu, DMAR_ECAP_REG);
> + iommu->ndoms = cap_nd(iommu->cap);
> +
> + /* Print Capabilities & Extended Capabilities */
> + DPRINTF(0, " caps: %s%s%s%s%s%s%s%s%s%s%s\n",
> + iommu->cap & CAP_AFL ? "afl " : "", /* adv fault */
> + iommu->cap & CAP_RWBF ? "rwbf " : "", /* write-buffer flush */
> + iommu->cap & CAP_PLMR ? "plmr " : "", /* protected lo region
> */
> + iommu->cap & CAP_PHMR ? "phmr " : "", /* protected hi region
> */
> + iommu->cap & CAP_CM ? "cm " : "", /* caching mode */
> + iommu->cap & CAP_ZLR ? "zlr " : "", /* zero-length read */
> + iommu->cap & CAP_PSI ? "psi " : "", /* page invalidate */
> + iommu->cap & CAP_DWD ? "dwd " : "", /* write drain */
> + iommu->cap & CAP_DRD ? "drd " : "", /* read drain */
> + iommu->cap & CAP_FL1GP ? "Gb " : "", /* 1Gb pages */
> + iommu->cap & CAP_PI ? "pi " : ""); /* posted interrupts */
> + DPRINTF(0, " ecap: %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
> + iommu->ecap & ECAP_C ? "c " : "", /* coherent */
> + iommu->ecap & ECAP_QI ? "qi " : "", /* queued invalidate */
> + iommu->ecap & ECAP_DT ? "dt " : "", /* device iotlb */
> + iommu->ecap & ECAP_IR ? "ir " : "", /* intr remap */
> + iommu->ecap & ECAP_EIM ? "eim " : "", /* x2apic */
> + iommu->ecap & ECAP_PT ? "pt " : "", /* passthrough */
> + iommu->ecap & ECAP_SC ? "sc " : "", /* snoop control */
> + iommu->ecap & ECAP_ECS ? "ecs " : "", /* extended context */
> + iommu->ecap & ECAP_MTS ? "mts " : "", /* memory type */
> + iommu->ecap & ECAP_NEST ? "nest " : "", /* nested translations
> */
> + iommu->ecap & ECAP_DIS ? "dis " : "", /* deferred
> invalidation */
> + iommu->ecap & ECAP_PASID ? "pas " : "", /* pasid */
> + iommu->ecap & ECAP_PRS ? "prs " : "", /* page request */
> + iommu->ecap & ECAP_ERS ? "ers " : "", /* execute request */
> + iommu->ecap & ECAP_SRS ? "srs " : "", /* supervisor request */
> + iommu->ecap & ECAP_NWFS ? "nwfs " : "", /* no write flag */
> + iommu->ecap & ECAP_EAFS ? "eafs " : ""); /* extended accessed
> flag */
> +
> + mtx_init(&iommu->reg_lock, IPL_HIGH);
> +
> + /* Clear Interrupt Masking */
> + iommu_write_4(iommu, DMAR_FSTS_REG, FSTS_PFO | FSTS_PPF);
> +
> + iommu->intr = acpidmar_intr_establish(iommu, IPL_HIGH,
> + acpidmar_intr, iommu, "dmarintr");
> +
> + /* Enable interrupts */
> + sts = iommu_read_4(iommu, DMAR_FECTL_REG);
> + iommu_write_4(iommu, DMAR_FECTL_REG, sts & ~FECTL_IM);
> +
> + /* Allocate root pointer */
> + iommu->root = iommu_alloc_page(iommu, &paddr);
> + DPRINTF(0, "Allocated root pointer: pa:%.16llx va:%p\n",
> + (uint64_t)paddr, iommu->root);
> + iommu->rtaddr = paddr;
> + iommu_flush_write_buffer(iommu);
> + iommu_set_rtaddr(iommu, paddr);
> +
> +#if 0
> + if (iommu->ecap & ECAP_QI) {
> + /* Queued Invalidation support */
> + iommu->qi = iommu_alloc_page(iommu, &iommu->qip);
> + iommu_write_8(iommu, DMAR_IQT_REG, 0);
> + iommu_write_8(iommu, DMAR_IQA_REG, iommu->qip | IQA_QS_256);
> + }
> + if (iommu->ecap & ECAP_IR) {
> + /* Interrupt remapping support */
> + iommu_write_8(iommu, DMAR_IRTA_REG, 0);
> + }
> +#endif
> +
> + /* Calculate guest address width and supported guest widths */
> + gaw = -1;
> + iommu->mgaw = cap_mgaw(iommu->cap);
> + DPRINTF(0, "gaw: %d { ", iommu->mgaw);
> + for (i = 0; i < 5; i++) {
> + if (cap_sagaw(iommu->cap) & (1L << i)) {
> + gaw = VTD_LEVELTOAW(i);
> + DPRINTF(0, "%d ", gaw);
> + iommu->agaw = gaw;
> + }
> + }
> + DPRINTF(0, "}\n");
> +
> + /* Cache current status register bits */
> + sts = iommu_read_4(iommu, DMAR_GSTS_REG);
> + if (sts & GSTS_TES)
> + iommu->gcmd |= GCMD_TE;
> + if (sts & GSTS_QIES)
> + iommu->gcmd |= GCMD_QIE;
> + if (sts & GSTS_IRES)
> + iommu->gcmd |= GCMD_IRE;
> + DPRINTF(0, "gcmd: %x preset\n", iommu->gcmd);
> + acpidmar_intr(iommu);
> + return (0);
> +}
> +
> +/* Read/Write IOMMU register */
> +uint32_t
> +iommu_read_4(struct iommu_softc *iommu, int reg)
> +{
> + uint32_t v;
> +
> + v = bus_space_read_4(iommu->iot, iommu->ioh, reg);
> + return (v);
> +}
> +
> +
> +void
> +iommu_write_4(struct iommu_softc *iommu, int reg, uint32_t v)
> +{
> + bus_space_write_4(iommu->iot, iommu->ioh, reg, (uint32_t)v);
> +}
> +
> +uint64_t
> +iommu_read_8(struct iommu_softc *iommu, int reg)
> +{
> + uint64_t v;
> +
> + v = bus_space_read_8(iommu->iot, iommu->ioh, reg);
> + return (v);
> +}
> +
> +void
> +iommu_write_8(struct iommu_softc *iommu, int reg, uint64_t v)
> +{
> + bus_space_write_8(iommu->iot, iommu->ioh, reg, v);
> +}
> +
> +/* Check if a device is within a device scope */
> +int
> +acpidmar_match_devscope(struct devlist_head *devlist, pci_chipset_tag_t pc,
> + int sid)
> +{
> + struct dmar_devlist *ds;
> + int sub, sec, i;
> + int bus, dev, fun, sbus;
> + pcireg_t reg;
> + pcitag_t tag;
> +
> + sbus = sid_bus(sid);
> + TAILQ_FOREACH(ds, devlist, link) {
> + bus = ds->bus;
> + dev = ds->dp[0].device;
> + fun = ds->dp[0].function;
> + /* Walk PCI bridges in path */
> + for (i = 1; i < ds->ndp; i++) {
> + tag = pci_make_tag(pc, bus, dev, fun);
> + reg = pci_conf_read(pc, tag, PPB_REG_BUSINFO);
> + bus = PPB_BUSINFO_SECONDARY(reg);
> + dev = ds->dp[i].device;
> + fun = ds->dp[i].function;
> + }
> +
> + /* Check for device exact match */
> + if (sid == mksid(bus, dev, fun)) {
> + return DMAR_ENDPOINT;
> + }
> +
> + /* Check for device subtree match */
> + if (ds->type == DMAR_BRIDGE) {
> + tag = pci_make_tag(pc, bus, dev, fun);
> + reg = pci_conf_read(pc, tag, PPB_REG_BUSINFO);
> + sec = PPB_BUSINFO_SECONDARY(reg);
> + sub = PPB_BUSINFO_SUBORDINATE(reg);
> + if (sec <= sbus && sbus <= sub) {
> + return DMAR_BRIDGE;
> + }
> + }
> + }
> +
> + return (0);
> +}
> +
> +struct domain *
> +domain_create(struct iommu_softc *iommu, int did)
> +{
> + struct domain *dom;
> + int gaw;
> +
> + DPRINTF(0, "iommu%d: create domain: %.4x\n", iommu->id, did);
> + dom = malloc(sizeof(*dom), M_DEVBUF, M_ZERO | M_WAITOK);
> + dom->did = did;
> + dom->iommu = iommu;
> + dom->pte = iommu_alloc_page(iommu, &dom->ptep);
> + TAILQ_INIT(&dom->devices);
> +
> + /* Setup DMA */
> + dom->dmat._cookie = dom;
> + dom->dmat._dmamap_create = dmar_dmamap_create; /* nop */
> + dom->dmat._dmamap_destroy = dmar_dmamap_destroy; /* nop */
> + dom->dmat._dmamap_load = dmar_dmamap_load; /* lm */
> + dom->dmat._dmamap_load_mbuf = dmar_dmamap_load_mbuf; /* lm */
> + dom->dmat._dmamap_load_uio = dmar_dmamap_load_uio; /* lm */
> + dom->dmat._dmamap_load_raw = dmar_dmamap_load_raw; /* lm */
> + dom->dmat._dmamap_unload = dmar_dmamap_unload; /* um */
> + dom->dmat._dmamap_sync = dmar_dmamap_sync; /* lm */
> + dom->dmat._dmamem_alloc = dmar_dmamem_alloc; /* nop */
> + dom->dmat._dmamem_free = dmar_dmamem_free; /* nop */
> + dom->dmat._dmamem_map = dmar_dmamem_map; /* nop */
> + dom->dmat._dmamem_unmap = dmar_dmamem_unmap; /* nop */
> + dom->dmat._dmamem_mmap = dmar_dmamem_mmap;
> +
> + snprintf(dom->exname, sizeof(dom->exname), "did:%x.%.4x",
> + iommu->id, dom->did);
> +
> + /* Setup IOMMU address map */
> + gaw = min(iommu->agaw, iommu->mgaw);
> + dom->iovamap = extent_create(dom->exname, 1024*1024*16,
> + (1LL << gaw)-1,
> + M_DEVBUF, NULL, 0,
> + EX_WAITOK|EX_NOCOALESCE);
> +
> + /* Zero out MSI Interrupt region */
> + extent_alloc_region(dom->iovamap, MSI_BASE_ADDRESS, MSI_BASE_SIZE,
> + EX_WAITOK);
> + mtx_init(&dom->exlck, IPL_HIGH);
> +
> + TAILQ_INSERT_TAIL(&iommu->domains, dom, link);
> +
> + return dom;
> +}
> +
> +void
> +domain_add_device(struct domain *dom, int sid)
> +{
> + struct domain_dev *ddev;
> +
> + DPRINTF(0, "add %s to iommu%d.%.4x\n", dmar_bdf(sid), dom->iommu->id,
> dom->did);
> + ddev = malloc(sizeof(*ddev), M_DEVBUF, M_ZERO | M_WAITOK);
> + ddev->sid = sid;
> + TAILQ_INSERT_TAIL(&dom->devices, ddev, link);
> +
> + /* Should set context entry here?? */
> +}
> +
> +void
> +domain_remove_device(struct domain *dom, int sid)
> +{
> + struct domain_dev *ddev, *tmp;
> +
> + TAILQ_FOREACH_SAFE(ddev, &dom->devices, link, tmp) {
> + if (ddev->sid == sid) {
> + TAILQ_REMOVE(&dom->devices, ddev, link);
> + free(ddev, sizeof(*ddev), M_DEVBUF);
> + }
> + }
> +}
> +
> +/* Lookup domain by segment & source id (bus.device.function) */
> +struct domain *
> +domain_lookup(struct acpidmar_softc *sc, int segment, int sid)
> +{
> + struct iommu_softc *iommu;
> + struct domain_dev *ddev;
> + struct domain *dom;
> + int rc;
> +
> + if (sc == NULL) {
> + return NULL;
> + }
> +
> + /* Lookup IOMMU for this device */
> + TAILQ_FOREACH(iommu, &sc->sc_drhds, link) {
> + if (iommu->segment != segment)
> + continue;
> + /* Check for devscope match or catchall iommu */
> + rc = acpidmar_match_devscope(&iommu->devices, sc->sc_pc, sid);
> + if (rc != 0 || iommu->flags) {
> + break;
> + }
> + }
> + if (!iommu) {
> + printf("%s: no iommu found\n", dmar_bdf(sid));
> + return NULL;
> + }
> +
> + /* Search domain devices */
> + TAILQ_FOREACH(dom, &iommu->domains, link) {
> + TAILQ_FOREACH(ddev, &dom->devices, link) {
> + /* XXX: match all functions? */
> + if (ddev->sid == sid) {
> + return dom;
> + }
> + }
> + }
> + if (iommu->ndoms <= 2) {
> + /* Running out of domains.. create catchall domain */
> + if (!iommu->unity) {
> + iommu->unity = domain_create(iommu, 1);
> + }
> + dom = iommu->unity;
> + } else {
> + dom = domain_create(iommu, --iommu->ndoms);
> + }
> + if (!dom) {
> + printf("no domain here\n");
> + return NULL;
> + }
> +
> + /* Add device to domain */
> + domain_add_device(dom, sid);
> +
> + return dom;
> +}
> +
> +/* Map Guest Pages into IOMMU */
> +void
> +_iommu_map(void *dom, vaddr_t va, bus_addr_t gpa, bus_size_t len)
> +{
> + bus_size_t i;
> + paddr_t hpa;
> +
> + if (dom == NULL) {
> + return;
> + }
> + DPRINTF(1, "Mapping dma: %lx = %lx/%lx\n", va, gpa, len);
> + for (i = 0; i < len; i += PAGE_SIZE) {
> + hpa = 0;
> + pmap_extract(curproc->p_vmspace->vm_map.pmap, va, &hpa);
> + domain_map_page(dom, gpa, hpa, PTE_P | PTE_R | PTE_W);
> + gpa += PAGE_SIZE;
> + va += PAGE_SIZE;
> + }
> +}
> +
> +/* Find IOMMU for a given PCI device */
> +void
> +*_iommu_domain(int segment, int bus, int dev, int func, int *id)
> +{
> + struct domain *dom;
> +
> + dom = domain_lookup(acpidmar_sc, segment, mksid(bus, dev, func));
> + if (dom) {
> + *id = dom->did;
> + }
> + return dom;
> +}
> +
> +void
> +domain_map_device(struct domain *dom, int sid);
> +
> +void
> +domain_map_device(struct domain *dom, int sid)
> +{
> + struct iommu_softc *iommu;
> + struct context_entry *ctx;
> + paddr_t paddr;
> + int bus, devfn;
> + int tt, lvl;
> +
> + iommu = dom->iommu;
> +
> + bus = sid_bus(sid);
> + devfn = sid_devfn(sid);
> + /* AMD attach device */
> + if (iommu->dte) {
> + struct ivhd_dte *dte = &iommu->dte[sid];
> + if (!dte->dw0) {
> + /* Setup Device Table Entry: bus.devfn */
> + DPRINTF(1, "@@@ PCI Attach: %.4x[%s] %.4x\n", sid,
> dmar_bdf(sid), dom->did);
> + dte_set_host_page_table_root_ptr(dte, dom->ptep);
> + dte_set_domain(dte, dom->did);
> + dte_set_mode(dte, 3); /* Set 3 level PTE */
> + dte_set_tv(dte);
> + dte_set_valid(dte);
> + ivhd_flush_devtab(iommu, dom->did);
> +#ifdef IOMMU_DEBUG
> + //ivhd_showreg(iommu);
> + ivhd_showdte(iommu);
> +#endif
> + }
> + return;
> + }
> +
> + /* Create Bus mapping */
> + if (!root_entry_is_valid(&iommu->root[bus])) {
> + iommu->ctx[bus] = iommu_alloc_page(iommu, &paddr);
> + iommu->root[bus].lo = paddr | ROOT_P;
> + iommu_flush_cache(iommu, &iommu->root[bus],
> + sizeof(struct root_entry));
> + DPRINTF(0, "iommu%d: Allocate context for bus: %.2x pa:%.16llx
> va:%p\n",
> + iommu->id, bus, (uint64_t)paddr,
> + iommu->ctx[bus]);
> + }
> +
> + /* Create DevFn mapping */
> + ctx = iommu->ctx[bus] + devfn;
> + if (!context_entry_is_valid(ctx)) {
> + tt = CTX_T_MULTI;
> + lvl = VTD_AWTOLEVEL(iommu->agaw);
> +
> + /* Initialize context */
> + context_set_slpte(ctx, dom->ptep);
> + context_set_translation_type(ctx, tt);
> + context_set_domain_id(ctx, dom->did);
> + context_set_address_width(ctx, lvl);
> + context_set_present(ctx);
> +
> + /* Flush it */
> + iommu_flush_cache(iommu, ctx, sizeof(struct context_entry));
> + if ((iommu->cap & CAP_CM) || acpidmar_force_cm) {
> + iommu_flush_ctx(iommu, CTX_DEVICE, dom->did, sid, 0);
> + iommu_flush_tlb(iommu, IOTLB_GLOBAL, 0);
> + } else {
> + iommu_flush_write_buffer(iommu);
> + }
> + DPRINTF(0, "iommu%d: %s set context ptep:%.16llx lvl:%d
> did:%.4x tt:%d\n",
> + iommu->id, dmar_bdf(sid), (uint64_t)dom->ptep, lvl,
> + dom->did, tt);
> + }
> +}
> +
> +struct domain *
> +acpidmar_pci_attach(struct acpidmar_softc *sc, int segment, int sid, int
> mapctx)
> +{
> + static struct domain *dom;
> +
> + dom = domain_lookup(sc, segment, sid);
> + if (!dom) {
> + printf("no domain: %s\n", dmar_bdf(sid));
> + return NULL;
> + }
> +
> + if (mapctx) {
> + domain_map_device(dom, sid);
> + }
> +
> + return dom;
> +}
> +
> +void
> +acpidmar_pci_hook(pci_chipset_tag_t pc, struct pci_attach_args *pa)
> +{
> + int bus, dev, fun, sid;
> + struct domain *dom;
> + pcireg_t reg;
> +
> + if (!acpidmar_sc) {
> + /* No DMAR, ignore */
> + return;
> + }
> +
> + /* Add device to our list if valid */
> + pci_decompose_tag(pc, pa->pa_tag, &bus, &dev, &fun);
> + sid = mksid(bus, dev, fun);
> + if (sid_flag[sid] & SID_INVALID)
> + return;
> +
> + reg = pci_conf_read(pc, pa->pa_tag, PCI_CLASS_REG);
> +
> + /* Add device to domain */
> + dom = acpidmar_pci_attach(acpidmar_sc, pa->pa_domain, sid, 0);
> + if (dom == NULL)
> + return;
> +
> + if (PCI_CLASS(reg) == PCI_CLASS_DISPLAY &&
> + PCI_SUBCLASS(reg) == PCI_SUBCLASS_DISPLAY_VGA) {
> + dom->flag = DOM_NOMAP;
> + }
> + if (PCI_CLASS(reg) == PCI_CLASS_BRIDGE &&
> + PCI_SUBCLASS(reg) == PCI_SUBCLASS_BRIDGE_ISA) {
> + /* For ISA Bridges, map 0-16Mb as 1:1 */
> + printf("dmar: %.4x:%.2x:%.2x.%x mapping ISA\n",
> + pa->pa_domain, bus, dev, fun);
> + domain_map_pthru(dom, 0x00, 16*1024*1024);
> + }
> +
> + /* Change DMA tag */
> + pa->pa_dmat = &dom->dmat;
> +}
> +
> +/* Create list of device scope entries from ACPI table */
> +void
> +acpidmar_parse_devscope(union acpidmar_entry *de, int off, int segment,
> + struct devlist_head *devlist)
> +{
> + struct acpidmar_devscope *ds;
> + struct dmar_devlist *d;
> + int dplen, i;
> +
> + TAILQ_INIT(devlist);
> + while (off < de->length) {
> + ds = (struct acpidmar_devscope *)((unsigned char *)de + off);
> + off += ds->length;
> +
> + /* We only care about bridges and endpoints */
> + if (ds->type != DMAR_ENDPOINT && ds->type != DMAR_BRIDGE)
> + continue;
> +
> + dplen = ds->length - sizeof(*ds);
> + d = malloc(sizeof(*d) + dplen, M_DEVBUF, M_ZERO | M_WAITOK);
> + d->bus = ds->bus;
> + d->type = ds->type;
> + d->ndp = dplen / 2;
> + d->dp = (void *)&d[1];
> + memcpy(d->dp, &ds[1], dplen);
> + TAILQ_INSERT_TAIL(devlist, d, link);
> +
> + DPRINTF(1, " %8s %.4x:%.2x.%.2x.%x {",
> + ds->type == DMAR_BRIDGE ? "bridge" : "endpoint",
> + segment, ds->bus,
> + d->dp[0].device,
> + d->dp[0].function);
> +
> + for (i = 1; i < d->ndp; i++) {
> + DPRINTF(1, " %2x.%x ",
> + d->dp[i].device,
> + d->dp[i].function);
> + }
> + DPRINTF(1, "}\n");
> + }
> +}
> +
> +/* DMA Remapping Hardware Unit */
> +void
> +acpidmar_drhd(struct acpidmar_softc *sc, union acpidmar_entry *de)
> +{
> + struct iommu_softc *iommu;
> +
> + printf("DRHD: segment:%.4x base:%.16llx flags:%.2x\n",
> + de->drhd.segment,
> + de->drhd.address,
> + de->drhd.flags);
> + iommu = malloc(sizeof(*iommu), M_DEVBUF, M_ZERO | M_WAITOK);
> + acpidmar_parse_devscope(de, sizeof(de->drhd), de->drhd.segment,
> + &iommu->devices);
> + iommu_init(sc, iommu, &de->drhd);
> +
> + if (de->drhd.flags) {
> + /* Catchall IOMMU goes at end of list */
> + TAILQ_INSERT_TAIL(&sc->sc_drhds, iommu, link);
> + } else {
> + TAILQ_INSERT_HEAD(&sc->sc_drhds, iommu, link);
> + }
> +}
> +
> +/* Reserved Memory Region Reporting */
> +void
> +acpidmar_rmrr(struct acpidmar_softc *sc, union acpidmar_entry *de)
> +{
> + struct rmrr_softc *rmrr;
> + bios_memmap_t *im, *jm;
> + uint64_t start, end;
> +
> + printf("RMRR: segment:%.4x range:%.16llx-%.16llx\n",
> + de->rmrr.segment, de->rmrr.base, de->rmrr.limit);
> + if (de->rmrr.limit <= de->rmrr.base) {
> + printf(" buggy BIOS\n");
> + return;
> + }
> +
> + rmrr = malloc(sizeof(*rmrr), M_DEVBUF, M_ZERO | M_WAITOK);
> + rmrr->start = trunc_page(de->rmrr.base);
> + rmrr->end = round_page(de->rmrr.limit);
> + rmrr->segment = de->rmrr.segment;
> + acpidmar_parse_devscope(de, sizeof(de->rmrr), de->rmrr.segment,
> + &rmrr->devices);
> +
> + for (im = bios_memmap; im->type != BIOS_MAP_END; im++) {
> + if (im->type != BIOS_MAP_RES)
> + continue;
> + /* Search for adjacent reserved regions */
> + start = im->addr;
> + end = im->addr+im->size;
> + for (jm = im+1; jm->type == BIOS_MAP_RES && end == jm->addr;
> + jm++) {
> + end = jm->addr+jm->size;
> + }
> + printf("e820: %.16llx - %.16llx\n", start, end);
> + if (start <= rmrr->start && rmrr->end <= end) {
> + /* Bah.. some buggy BIOS stomp outside RMRR */
> + printf(" ** inside E820 Reserved %.16llx %.16llx\n",
> + start, end);
> + rmrr->start = trunc_page(start);
> + rmrr->end = round_page(end);
> + break;
> + }
> + }
> + TAILQ_INSERT_TAIL(&sc->sc_rmrrs, rmrr, link);
> +}
> +
> +/* Root Port ATS Reporting */
> +void
> +acpidmar_atsr(struct acpidmar_softc *sc, union acpidmar_entry *de)
> +{
> + struct atsr_softc *atsr;
> +
> + printf("ATSR: segment:%.4x flags:%x\n",
> + de->atsr.segment,
> + de->atsr.flags);
> +
> + atsr = malloc(sizeof(*atsr), M_DEVBUF, M_ZERO | M_WAITOK);
> + atsr->flags = de->atsr.flags;
> + atsr->segment = de->atsr.segment;
> + acpidmar_parse_devscope(de, sizeof(de->atsr), de->atsr.segment,
> + &atsr->devices);
> +
> + TAILQ_INSERT_TAIL(&sc->sc_atsrs, atsr, link);
> +}
> +
> +void
> +acpidmar_init(struct acpidmar_softc *sc, struct acpi_dmar *dmar)
> +{
> + struct rmrr_softc *rmrr;
> + struct iommu_softc *iommu;
> + struct domain *dom;
> + struct dmar_devlist *dl;
> + union acpidmar_entry *de;
> + int off, sid, rc;
> +
> + domain_map_page = domain_map_page_intel;
> + printf(": hardware width: %d, intr_remap:%d x2apic_opt_out:%d\n",
> + dmar->haw+1,
> + !!(dmar->flags & 0x1),
> + !!(dmar->flags & 0x2));
> + sc->sc_haw = dmar->haw+1;
> + sc->sc_flags = dmar->flags;
> +
> + TAILQ_INIT(&sc->sc_drhds);
> + TAILQ_INIT(&sc->sc_rmrrs);
> + TAILQ_INIT(&sc->sc_atsrs);
> +
> + off = sizeof(*dmar);
> + while (off < dmar->hdr.length) {
> + de = (union acpidmar_entry *)((unsigned char *)dmar + off);
> + switch (de->type) {
> + case DMAR_DRHD:
> + acpidmar_drhd(sc, de);
> + break;
> + case DMAR_RMRR:
> + acpidmar_rmrr(sc, de);
> + break;
> + case DMAR_ATSR:
> + acpidmar_atsr(sc, de);
> + break;
> + default:
> + printf("DMAR: unknown %x\n", de->type);
> + break;
> + }
> + off += de->length;
> + }
> +
> + /* Pre-create domains for iommu devices */
> + TAILQ_FOREACH(iommu, &sc->sc_drhds, link) {
> + TAILQ_FOREACH(dl, &iommu->devices, link) {
> + sid = mksid(dl->bus, dl->dp[0].device,
> + dl->dp[0].function);
> + dom = acpidmar_pci_attach(sc, iommu->segment, sid, 0);
> + if (dom != NULL) {
> + printf("%.4x:%.2x:%.2x.%x iommu:%d did:%.4x\n",
> + iommu->segment, dl->bus, dl->dp[0].device,
> dl->dp[0].function,
> + iommu->id, dom->did);
> + }
> + }
> + }
> + /* Map passthrough pages for RMRR */
> + TAILQ_FOREACH(rmrr, &sc->sc_rmrrs, link) {
> + TAILQ_FOREACH(dl, &rmrr->devices, link) {
> + sid = mksid(dl->bus, dl->dp[0].device,
> + dl->dp[0].function);
> + dom = acpidmar_pci_attach(sc, rmrr->segment, sid, 0);
> + if (dom != NULL) {
> + printf("%s map ident: %.16llx %.16llx\n",
> + dom_bdf(dom), rmrr->start, rmrr->end);
> + domain_map_pthru(dom, rmrr->start, rmrr->end);
> + rc = extent_alloc_region(dom->iovamap,
> + rmrr->start, rmrr->end, EX_WAITOK);
> + }
> + }
> + }
> +}
> +
> +
> +/*=====================================================
> + * AMD Vi
> + *=====================================================*/
> +void acpiivrs_ivhd(struct acpidmar_softc *, struct acpi_ivhd *);
> +int ivhd_iommu_init(struct acpidmar_softc *, struct iommu_softc *,
> + struct acpi_ivhd *);
> +int _ivhd_issue_command(struct iommu_softc *, const struct ivhd_command *);
> +void ivhd_show_event(struct iommu_softc *, struct ivhd_event *evt,
> int);
> +int ivhd_issue_command(struct iommu_softc *, const struct ivhd_command *,
> int);
> +int ivhd_invalidate_domain(struct iommu_softc *, int);
> +void ivhd_intr_map(struct iommu_softc *, int);
> +void ivhd_checkerr(struct iommu_softc *iommu);
> +int acpiivhd_intr(void *);
> +
> +int
> +acpiivhd_intr(void *ctx)
> +{
> + struct iommu_softc *iommu = ctx;
> +
> + if (!iommu->dte)
> + return (0);
> + ivhd_poll_events(iommu);
> + return (1);
> +}
> +
> +/* Setup interrupt for AMD */
> +void
> +ivhd_intr_map(struct iommu_softc *iommu, int devid) {
> + pci_intr_handle_t ih;
> +
> + if (iommu->intr)
> + return;
> + ih.tag = pci_make_tag(NULL, sid_bus(devid), sid_dev(devid),
> sid_fun(devid));
> + ih.line = APIC_INT_VIA_MSG;
> + ih.pin = 0;
> + iommu->intr = pci_intr_establish(NULL, ih, IPL_NET | IPL_MPSAFE,
> + acpiivhd_intr, iommu, "amd_iommu");
> + printf("amd iommu intr: %p\n", iommu->intr);
> +}
> +
> +void
> +_dumppte(struct pte_entry *pte, int lvl, vaddr_t va)
> +{
> + char *pfx[] = { " ", " ", " ", " ", "" };
> + uint64_t i, sh;
> + struct pte_entry *npte;
> +
> + for (i = 0; i < 512; i++) {
> + sh = (i << (((lvl-1) * 9) + 12));
> + if (pte[i].val & PTE_P) {
> + if (lvl > 1) {
> + npte = (void *)PMAP_DIRECT_MAP((pte[i].val &
> PTE_PADDR_MASK));
> + printf("%slvl%d: %.16llx nxt:%llu\n", pfx[lvl],
> lvl,
> + pte[i].val, (pte[i].val >> 9) & 7);
> + _dumppte(npte, lvl-1, va | sh);
> + }
> + else {
> + printf("%slvl%d: %.16llx <- %.16llx \n",
> pfx[lvl], lvl,
> + pte[i].val, va | sh);
> + }
> + }
> + }
> +}
> +
> +void
> +ivhd_showpage(struct iommu_softc *iommu, int sid, paddr_t paddr)
> +{
> + struct domain *dom;
> + static int show = 0;
> +
> + if (show > 10)
> + return;
> + show++;
> + dom = acpidmar_pci_attach(acpidmar_sc, 0, sid, 0);
> + if (!dom)
> + return;
> + printf("DTE: %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n",
> + iommu->dte[sid].dw0,
> + iommu->dte[sid].dw1,
> + iommu->dte[sid].dw2,
> + iommu->dte[sid].dw3,
> + iommu->dte[sid].dw4,
> + iommu->dte[sid].dw5,
> + iommu->dte[sid].dw6,
> + iommu->dte[sid].dw7);
> + _dumppte(dom->pte, 3, 0);
> +}
> +
> +/* Display AMD IOMMU Error */
> +void
> +ivhd_show_event(struct iommu_softc *iommu, struct ivhd_event *evt, int head)
> +{
> + int type, sid, did, flag;
> + uint64_t address;
> +
> + /* Get Device, Domain, Address and Type of event */
> + sid = __EXTRACT(evt->dw0, EVT_SID);
> + type = __EXTRACT(evt->dw1, EVT_TYPE);
> + did = __EXTRACT(evt->dw1, EVT_DID);
> + flag = __EXTRACT(evt->dw1, EVT_FLAG);
> + address = _get64(&evt->dw2);
> +
> + printf("=== IOMMU Error[%.4x]: ", head);
> + switch (type) {
> + case ILLEGAL_DEV_TABLE_ENTRY:
> + printf("illegal dev table entry dev=%s addr=0x%.16llx %s, %s,
> %s, %s\n",
> + dmar_bdf(sid), address,
> + evt->dw1 & EVT_TR ? "translation" : "transaction",
> + evt->dw1 & EVT_RZ ? "reserved bit" : "invalid level",
> + evt->dw1 & EVT_RW ? "write" : "read",
> + evt->dw1 & EVT_I ? "interrupt" : "memory");
> + ivhd_showdte(iommu);
> + break;
> + case IO_PAGE_FAULT:
> + printf("io page fault dev=%s did=0x%.4x addr=0x%.16llx\n%s, %s,
> %s, %s, %s, %s\n",
> + dmar_bdf(sid), did, address,
> + evt->dw1 & EVT_TR ? "translation" : "transaction",
> + evt->dw1 & EVT_RZ ? "reserved bit" : "invalid level",
> + evt->dw1 & EVT_PE ? "no perm" : "perm",
> + evt->dw1 & EVT_RW ? "write" : "read",
> + evt->dw1 & EVT_PR ? "present" : "not present",
> + evt->dw1 & EVT_I ? "interrupt" : "memory");
> + ivhd_showdte(iommu);
> + ivhd_showpage(iommu, sid, address);
> + break;
> + case DEV_TAB_HARDWARE_ERROR:
> + printf("device table hardware error dev=%s addr=0x%.16llx %s,
> %s, %s\n",
> + dmar_bdf(sid), address,
> + evt->dw1 & EVT_TR ? "translation" : "transaction",
> + evt->dw1 & EVT_RW ? "write" : "read",
> + evt->dw1 & EVT_I ? "interrupt" : "memory");
> + ivhd_showdte(iommu);
> + break;
> + case PAGE_TAB_HARDWARE_ERROR:
> + printf("page table hardware error dev=%s addr=0x%.16llx %s, %s,
> %s\n",
> + dmar_bdf(sid), address,
> + evt->dw1 & EVT_TR ? "translation" : "transaction",
> + evt->dw1 & EVT_RW ? "write" : "read",
> + evt->dw1 & EVT_I ? "interrupt" : "memory");
> + ivhd_showdte(iommu);
> + break;
> + case ILLEGAL_COMMAND_ERROR:
> + printf("illegal command addr=0x%.16llx\n", address);
> + ivhd_showcmd(iommu);
> + break;
> + case COMMAND_HARDWARE_ERROR:
> + printf("command hardware error addr=0x%.16llx flag=0x%.4x\n",
> + address, flag);
> + ivhd_showcmd(iommu);
> + break;
> + case IOTLB_INV_TIMEOUT:
> + printf("iotlb invalidation timeout dev=%s address=0x%.16llx\n",
> + dmar_bdf(sid), address);
> + break;
> + case INVALID_DEVICE_REQUEST:
> + printf("invalid device request dev=%s addr=0x%.16llx
> flag=0x%.4x\n",
> + dmar_bdf(sid), address, flag);
> + break;
> + default:
> + printf("unknown type=0x%.2x\n", type);
> + break;
> + }
> + /* Clear old event */
> + evt->dw0 = 0;
> + evt->dw1 = 0;
> + evt->dw2 = 0;
> + evt->dw3 = 0;
> +}
> +
> +/* AMD: Process IOMMU error from hardware */
> +int
> +ivhd_poll_events(struct iommu_softc *iommu)
> +{
> + uint32_t head, tail;
> + int sz;
> +
> + sz = sizeof(struct ivhd_event);
> + head = iommu_read_4(iommu, EVT_HEAD_REG);
> + tail = iommu_read_4(iommu, EVT_TAIL_REG);
> + if (head == tail) {
> + /* No pending events */
> + return (0);
> + }
> + while (head != tail) {
> + ivhd_show_event(iommu, iommu->evt_tbl + head, head);
> + head = (head + sz) % EVT_TBL_SIZE;
> + }
> + iommu_write_4(iommu, EVT_HEAD_REG, head);
> + return (0);
> +}
> +
> +/* AMD: Issue command to IOMMU queue */
> +int
> +_ivhd_issue_command(struct iommu_softc *iommu, const struct ivhd_command
> *cmd)
> +{
> + u_long rf;
> + uint32_t head, tail, next;
> + int sz;
> +
> + head = iommu_read_4(iommu, CMD_HEAD_REG);
> + sz = sizeof(*cmd);
> + rf = intr_disable();
> + tail = iommu_read_4(iommu, CMD_TAIL_REG);
> + next = (tail + sz) % CMD_TBL_SIZE;
> + if (next == head) {
> + printf("FULL\n");
> + /* Queue is full */
> + intr_restore(rf);
> + return -EBUSY;
> + }
> + memcpy(iommu->cmd_tbl + tail, cmd, sz);
> + iommu_write_4(iommu, CMD_TAIL_REG, next);
> + intr_restore(rf);
> + return (tail / sz);
> +}
> +
> +#define IVHD_MAXDELAY 8
> +
> +int
> +ivhd_issue_command(struct iommu_softc *iommu, const struct ivhd_command
> *cmd, int wait)
> +{
> + struct ivhd_command wq = { 0 };
> + volatile uint64_t wv __aligned(16) = 0LL;
> + paddr_t paddr;
> + int rc, i;
> +
> + rc = _ivhd_issue_command(iommu, cmd);
> + if (rc >= 0 && wait) {
> + /* Wait for previous commands to complete.
> + * Store address of completion variable to command */
> + pmap_extract(pmap_kernel(), (vaddr_t)&wv, &paddr);
> + wq.dw0 = (paddr & ~0xF) | 0x1;
> + wq.dw1 = (COMPLETION_WAIT << CMD_SHIFT) | ((paddr >> 32) &
> 0xFFFFF);
> + wq.dw2 = 0xDEADBEEF;
> + wq.dw3 = 0xFEEDC0DE;
> +
> + rc = _ivhd_issue_command(iommu, &wq);
> + /* wv will change to value in dw2/dw3 when command is complete
> */
> + for (i = 0; i < IVHD_MAXDELAY && !wv; i++) {
> + DELAY(10 << i);
> + }
> + if (i == IVHD_MAXDELAY) {
> + printf("ivhd command timeout: %.8x %.8x %.8x %.8x
> wv:%llx idx:%x\n",
> + cmd->dw0, cmd->dw1, cmd->dw2, cmd->dw3, wv, rc);
> + }
> + }
> + return rc;
> +
> +}
> +
> +/* AMD: Flush changes to Device Table Entry for a specific domain */
> +int
> +ivhd_flush_devtab(struct iommu_softc *iommu, int did)
> +{
> + struct ivhd_command cmd = { .dw0 = did, .dw1 = INVALIDATE_DEVTAB_ENTRY
> << CMD_SHIFT };
> + return ivhd_issue_command(iommu, &cmd, 1);
> +}
> +
> +/* AMD: Invalidate all IOMMU device and page tables */
> +int
> +ivhd_invalidate_iommu_all(struct iommu_softc *iommu)
> +{
> + struct ivhd_command cmd = { .dw1 = INVALIDATE_IOMMU_ALL << CMD_SHIFT };
> + return ivhd_issue_command(iommu, &cmd, 0);
> +}
> +
> +/* AMD: Invalidate interrupt remapping */
> +int
> +ivhd_invalidate_interrupt_table(struct iommu_softc *iommu, int did)
> +{
> + struct ivhd_command cmd = { .dw0 = did, .dw1 =
> INVALIDATE_INTERRUPT_TABLE << CMD_SHIFT };
> + return ivhd_issue_command(iommu, &cmd, 0);
> +}
> +
> +/* AMD: Invalidate all page tables in a domain */
> +int
> +ivhd_invalidate_domain(struct iommu_softc *iommu, int did)
> +{
> + struct ivhd_command cmd = { .dw1 = did | (INVALIDATE_IOMMU_PAGES <<
> CMD_SHIFT) };
> +
> + cmd.dw2 = 0xFFFFF000 | 0x3;
> + cmd.dw3 = 0x7FFFFFFF;
> + return ivhd_issue_command(iommu, &cmd, 1);
> +}
> +
> +/* AMD: Display Registers */
> +void
> +ivhd_showreg(struct iommu_softc *iommu)
> +{
> + printf("---- dt:%.16llx cmd:%.16llx evt:%.16llx ctl:%.16llx
> sts:%.16llx\n",
> + iommu_read_8(iommu, DEV_TAB_BASE_REG),
> + iommu_read_8(iommu, CMD_BASE_REG),
> + iommu_read_8(iommu, EVT_BASE_REG),
> + iommu_read_8(iommu, IOMMUCTL_REG),
> + iommu_read_8(iommu, IOMMUSTS_REG));
> + printf("---- cmd queue:%.16llx %.16llx evt queue:%.16llx %.16llx\n",
> + iommu_read_8(iommu, CMD_HEAD_REG),
> + iommu_read_8(iommu, CMD_TAIL_REG),
> + iommu_read_8(iommu, EVT_HEAD_REG),
> + iommu_read_8(iommu, EVT_TAIL_REG));
> +}
> +
> +/* AMD: Generate Errors to test event handler */
> +void
> +ivhd_checkerr(struct iommu_softc *iommu)
> +{
> + struct ivhd_command cmd = { -1, -1, -1, -1 };
> +
> + /* Generate ILLEGAL DEV TAB entry? */
> + iommu->dte[0x2303].dw0 = -1; /* invalid */
> + iommu->dte[0x2303].dw2 = 0x1234; /* domain */
> + iommu->dte[0x2303].dw7 = -1; /* reserved */
> + ivhd_flush_devtab(iommu, 0x1234);
> + ivhd_poll_events(iommu);
> +
> + /* Generate ILLEGAL_COMMAND_ERROR : ok */
> + ivhd_issue_command(iommu, &cmd, 0);
> + ivhd_poll_events(iommu);
> +
> + /* Generate page hardware error */
> +}
> +
> +/* AMD: Show Device Table Entry */
> +void
> +ivhd_showdte(struct iommu_softc *iommu)
> +{
> + int i;
> +
> +
> + for (i = 0; i < 65536; i++) {
> + if (iommu->dte[i].dw0) {
> + printf("%.2x:%.2x.%x: %.8x %.8x %.8x %.8x %.8x %.8x
> %.8x %.8x\n",
> + i >> 8, (i >> 3) & 0x1F, i & 0x7,
> + iommu->dte[i].dw0, iommu->dte[i].dw1,
> + iommu->dte[i].dw2, iommu->dte[i].dw3,
> + iommu->dte[i].dw4, iommu->dte[i].dw5,
> + iommu->dte[i].dw6, iommu->dte[i].dw7);
> + }
> + }
> +}
> +
> +/* AMD: Show command entries */
> +void
> +ivhd_showcmd(struct iommu_softc *iommu)
> +{
> + struct ivhd_command *ihd;
> + paddr_t phd;
> + int i;
> +
> + ihd = iommu->cmd_tbl;
> + phd = iommu_read_8(iommu, CMD_BASE_REG) & CMD_BASE_MASK;
> + for (i = 0; i < 4096 / 128; i++) {
> + printf("%.2x: %.16llx %.8x %.8x %.8x %.8x\n", i,
> + (uint64_t)phd + i * sizeof(*ihd),
> + ihd[i].dw0,ihd[i].dw1,ihd[i].dw2,ihd[i].dw3);
> + }
> +}
> +
> +#define _c(x) (int)((iommu->ecap >> x ##_SHIFT) & x ## _MASK)
> +
> +/* AMD: Initialize IOMMU */
> +int
> +ivhd_iommu_init(struct acpidmar_softc *sc, struct iommu_softc *iommu,
> + struct acpi_ivhd *ivhd)
> +{
> + static int niommu;
> + paddr_t paddr;
> + uint64_t ov;
> +
> + if (sc == NULL || iommu == NULL || ivhd == NULL) {
> + printf("Bad pointer to iommu_init!\n");
> + return -1;
> + }
> + if (_bus_space_map(sc->sc_memt, ivhd->address, 0x80000, 0, &iommu->ioh)
> != 0) {
> + printf("Bus Space Map fails\n");
> + return -1;
> + }
> + TAILQ_INIT(&iommu->domains);
> + TAILQ_INIT(&iommu->devices);
> +
> + /* Setup address width and number of domains */
> + iommu->id = ++niommu;
> + iommu->iot = sc->sc_memt;
> + iommu->mgaw = 48;
> + iommu->agaw = 48;
> + iommu->flags = 1;
> + iommu->segment = 0;
> + iommu->ndoms = 256;
> +
> + printf(": AMD iommu%d at 0x%.8llx\n", iommu->id, ivhd->address);
> +
> + iommu->ecap = iommu_read_8(iommu, EXTFEAT_REG);
> + DPRINTF("iommu%d: ecap:%.16llx ", iommu->id, iommu->ecap);
> + DPRINTF("%s%s%s%s%s%s%s%s\n",
> + iommu->ecap & EFR_PREFSUP ? "pref " : "",
> + iommu->ecap & EFR_PPRSUP ? "ppr " : "",
> + iommu->ecap & EFR_NXSUP ? "nx " : "",
> + iommu->ecap & EFR_GTSUP ? "gt " : "",
> + iommu->ecap & EFR_IASUP ? "ia " : "",
> + iommu->ecap & EFR_GASUP ? "ga " : "",
> + iommu->ecap & EFR_HESUP ? "he " : "",
> + iommu->ecap & EFR_PCSUP ? "pc " : "");
> + DPRINTF(0,"hats:%x gats:%x glxsup:%x smif:%x smifrc:%x gam:%x\n",
> + _c(EFR_HATS), _c(EFR_GATS), _c(EFR_GLXSUP), _c(EFR_SMIFSUP),
> + _c(EFR_SMIFRC), _c(EFR_GAMSUP));
> +
> + /* Turn off iommu */
> + ov = iommu_read_8(iommu, IOMMUCTL_REG);
> + iommu_write_8(iommu, IOMMUCTL_REG, ov & ~(CTL_IOMMUEN | CTL_COHERENT |
> + CTL_HTTUNEN | CTL_RESPASSPW | CTL_PASSPW | CTL_ISOC));
> +
> + /* Enable intr, mark IOMMU device as invalid for remap */
> + sid_flag[ivhd->devid] |= SID_INVALID;
> + ivhd_intr_map(iommu, ivhd->devid);
> +
> + /* Setup command buffer with 4k buffer (128 entries) */
> + iommu->cmd_tbl = iommu_alloc_page(iommu, &paddr);
> + iommu_write_8(iommu, CMD_BASE_REG, (paddr & CMD_BASE_MASK) |
> CMD_TBL_LEN_4K);
> + iommu_write_4(iommu, CMD_HEAD_REG, 0x00);
> + iommu_write_4(iommu, CMD_TAIL_REG, 0x00);
> + iommu->cmd_tblp = paddr;
> +
> + /* Setup event log with 4k buffer (128 entries) */
> + iommu->evt_tbl = iommu_alloc_page(iommu, &paddr);
> + iommu_write_8(iommu, EVT_BASE_REG, (paddr & EVT_BASE_MASK) |
> EVT_TBL_LEN_4K);
> + iommu_write_4(iommu, EVT_HEAD_REG, 0x00);
> + iommu_write_4(iommu, EVT_TAIL_REG, 0x00);
> + iommu->evt_tblp = paddr;
> +
> + /* Setup device table
> + * 1 entry per source ID (bus:device:function - 64k entries)
> + */
> + iommu->dte = sc->sc_hwdte;
> + iommu_write_8(iommu, DEV_TAB_BASE_REG, (sc->sc_hwdtep & DEV_TAB_MASK) |
> DEV_TAB_LEN);
> +
> + /* Enable IOMMU */
> + ov |= (CTL_IOMMUEN | CTL_EVENTLOGEN | CTL_CMDBUFEN | CTL_EVENTINTEN);
> + if (ivhd->flags & IVHD_COHERENT)
> + ov |= CTL_COHERENT;
> + if (ivhd->flags & IVHD_HTTUNEN)
> + ov |= CTL_HTTUNEN;
> + if (ivhd->flags & IVHD_RESPASSPW)
> + ov |= CTL_RESPASSPW;
> + if (ivhd->flags & IVHD_PASSPW)
> + ov |= CTL_PASSPW;
> + if (ivhd->flags & IVHD_ISOC)
> + ov |= CTL_ISOC;
> + ov &= ~(CTL_INVTIMEOUT_MASK << CTL_INVTIMEOUT_SHIFT);
> + ov |= (CTL_INVTIMEOUT_10MS << CTL_INVTIMEOUT_SHIFT);
> + iommu_write_8(iommu, IOMMUCTL_REG, ov);
> +
> + ivhd_invalidate_iommu_all(iommu);
> +
> + TAILQ_INSERT_TAIL(&sc->sc_drhds, iommu, link);
> + return 0;
> +}
> +
> +void
> +acpiivrs_ivhd(struct acpidmar_softc *sc, struct acpi_ivhd *ivhd)
> +{
> + struct iommu_softc *iommu;
> + struct acpi_ivhd_ext *ext;
> + union acpi_ivhd_entry *ie;
> + int start, off, dte, all_dte = 0;
> +
> + if (ivhd->type == IVRS_IVHD_EXT) {
> + ext = (struct acpi_ivhd_ext *)ivhd;
> + DPRINTF(0,"ivhd: %.2x %.2x %.4x %.4x:%s %.4x %.16llx %.4x %.8x
> %.16llx\n",
> + ext->type, ext->flags, ext->length,
> + ext->segment, dmar_bdf(ext->devid), ext->cap,
> + ext->address, ext->info,
> + ext->attrib, ext->efr);
> + if (ext->flags & IVHD_PPRSUP)
> + DPRINTF(0," PPRSup");
> + if (ext->flags & IVHD_PREFSUP)
> + DPRINTF(0," PreFSup");
> + if (ext->flags & IVHD_COHERENT)
> + DPRINTF(0," Coherent");
> + if (ext->flags & IVHD_IOTLB)
> + DPRINTF(0," Iotlb");
> + if (ext->flags & IVHD_ISOC)
> + DPRINTF(0," ISoc");
> + if (ext->flags & IVHD_RESPASSPW)
> + DPRINTF(0," ResPassPW");
> + if (ext->flags & IVHD_PASSPW)
> + DPRINTF(0," PassPW");
> + if (ext->flags & IVHD_HTTUNEN)
> + DPRINTF(0, " HtTunEn");
> + if (ext->flags)
> + DPRINTF(0,"\n");
> + off = sizeof(*ext);
> + iommu = malloc(sizeof(*iommu), M_DEVBUF, M_ZERO|M_WAITOK);
> + ivhd_iommu_init(sc, iommu, ivhd);
> + } else {
> + DPRINTF(0,"ivhd: %.2x %.2x %.4x %.4x:%s %.4x %.16llx %.4x
> %.8x\n",
> + ivhd->type, ivhd->flags, ivhd->length,
> + ivhd->segment, dmar_bdf(ivhd->devid), ivhd->cap,
> + ivhd->address, ivhd->info,
> + ivhd->feature);
> + if (ivhd->flags & IVHD_PPRSUP)
> + DPRINTF(0," PPRSup");
> + if (ivhd->flags & IVHD_PREFSUP)
> + DPRINTF(0," PreFSup");
> + if (ivhd->flags & IVHD_COHERENT)
> + DPRINTF(0," Coherent");
> + if (ivhd->flags & IVHD_IOTLB)
> + DPRINTF(0," Iotlb");
> + if (ivhd->flags & IVHD_ISOC)
> + DPRINTF(0," ISoc");
> + if (ivhd->flags & IVHD_RESPASSPW)
> + DPRINTF(0," ResPassPW");
> + if (ivhd->flags & IVHD_PASSPW)
> + DPRINTF(0," PassPW");
> + if (ivhd->flags & IVHD_HTTUNEN)
> + DPRINTF(0, " HtTunEn");
> + if (ivhd->flags)
> + DPRINTF(0,"\n");
> + off = sizeof(*ivhd);
> + }
> + while (off < ivhd->length) {
> + ie = (void *)ivhd + off;
> + switch (ie->type) {
> + case IVHD_ALL:
> + all_dte = ie->all.data;
> + DPRINTF(0," ALL %.4x\n", dte);
> + off += sizeof(ie->all);
> + break;
> + case IVHD_SEL:
> + dte = ie->sel.data;
> + DPRINTF(0," SELECT: %s %.4x\n",
> dmar_bdf(ie->sel.devid), dte);
> + off += sizeof(ie->sel);
> + break;
> + case IVHD_SOR:
> + dte = ie->sor.data;
> + start = ie->sor.devid;
> + DPRINTF(0," SOR: %s %.4x\n", dmar_bdf(start), dte);
> + off += sizeof(ie->sor);
> + break;
> + case IVHD_EOR:
> + DPRINTF(0," EOR: %s\n", dmar_bdf(ie->eor.devid));
> + off += sizeof(ie->eor);
> + break;
> + case IVHD_ALIAS_SEL:
> + dte = ie->alias.data;
> + DPRINTF(0," ALIAS: src=%s: ",
> dmar_bdf(ie->alias.srcid));
> + DPRINTF(0," %s %.4x\n", dmar_bdf(ie->alias.devid), dte);
> + off += sizeof(ie->alias);
> + break;
> + case IVHD_ALIAS_SOR:
> + dte = ie->alias.data;
> + DPRINTF(0," ALIAS_SOR: %s %.4x ",
> dmar_bdf(ie->alias.devid), dte);
> + DPRINTF(0," src=%s\n", dmar_bdf(ie->alias.srcid));
> + off += sizeof(ie->alias);
> + break;
> + case IVHD_EXT_SEL:
> + dte = ie->ext.data;
> + DPRINTF(0," EXT SEL: %s %.4x %.8x\n",
> dmar_bdf(ie->ext.devid),
> + dte, ie->ext.extdata);
> + off += sizeof(ie->ext);
> + break;
> + case IVHD_EXT_SOR:
> + dte = ie->ext.data;
> + DPRINTF(0," EXT SOR: %s %.4x %.8x\n",
> dmar_bdf(ie->ext.devid),
> + dte, ie->ext.extdata);
> + off += sizeof(ie->ext);
> + break;
> + case IVHD_SPECIAL:
> + DPRINTF(0," SPECIAL\n");
> + off += sizeof(ie->special);
> + break;
> + default:
> + DPRINTF(0," 2:unknown %x\n", ie->type);
> + off = ivhd->length;
> + break;
> + }
> + }
> +}
> +
> +void
> +acpiivrs_init(struct acpidmar_softc *sc, struct acpi_ivrs *ivrs)
> +{
> + union acpi_ivrs_entry *ie;
> + int off;
> +
> + if (!sc->sc_hwdte) {
> + sc->sc_hwdte = iommu_alloc_hwdte(sc, HWDTE_SIZE,
> &sc->sc_hwdtep);
> + if (sc->sc_hwdte == NULL)
> + panic("Can't allocate HWDTE!\n");
> + }
> +
> + domain_map_page = domain_map_page_amd;
> + DPRINTF(0,"IVRS Version: %d\n", ivrs->hdr.revision);
> + DPRINTF(0," VA Size: %d\n", (ivrs->ivinfo >> IVRS_VASIZE_SHIFT) &
> IVRS_VASIZE_MASK);
> + DPRINTF(0," PA Size: %d\n", (ivrs->ivinfo >> IVRS_PASIZE_SHIFT) &
> IVRS_PASIZE_MASK);
> +
> + TAILQ_INIT(&sc->sc_drhds);
> + TAILQ_INIT(&sc->sc_rmrrs);
> + TAILQ_INIT(&sc->sc_atsrs);
> +
> + DPRINTF(0,"======== IVRS\n");
> + off = sizeof(*ivrs);
> + while (off < ivrs->hdr.length) {
> + ie = (void *)ivrs + off;
> + switch (ie->type) {
> + case IVRS_IVHD:
> + case IVRS_IVHD_EXT:
> + acpiivrs_ivhd(sc, &ie->ivhd);
> + break;
> + case IVRS_IVMD_ALL:
> + case IVRS_IVMD_SPECIFIED:
> + case IVRS_IVMD_RANGE:
> + DPRINTF(0,"ivmd\n");
> + break;
> + default:
> + DPRINTF(0,"1:unknown: %x\n", ie->type);
> + break;
> + }
> + off += ie->length;
> + }
> + DPRINTF(0,"======== End IVRS\n");
> +}
> +
> +static int
> +acpiivhd_activate(struct iommu_softc *iommu, int act)
> +{
> + switch (act) {
> + case DVACT_SUSPEND:
> + iommu->flags |= IOMMU_FLAGS_SUSPEND;
> + break;
> + case DVACT_RESUME:
> + iommu->flags &= ~IOMMU_FLAGS_SUSPEND;
> + break;
> + }
> + return (0);
> +}
> +
> +int
> +acpidmar_activate(struct device *self, int act)
> +{
> + struct acpidmar_softc *sc = (struct acpidmar_softc *)self;
> + struct iommu_softc *iommu;
> +
> + printf("called acpidmar_activate %d %p\n", act, sc);
> +
> + if (sc == NULL) {
> + return (0);
> + }
> +
> + switch (act) {
> + case DVACT_RESUME:
> + TAILQ_FOREACH(iommu, &sc->sc_drhds, link) {
> + printf("iommu%d resume\n", iommu->id);
> + if (iommu->dte) {
> + acpiivhd_activate(iommu, act);
> + continue;
> + }
> + iommu_flush_write_buffer(iommu);
> + iommu_set_rtaddr(iommu, iommu->rtaddr);
> + iommu_write_4(iommu, DMAR_FEDATA_REG, iommu->fedata);
> + iommu_write_4(iommu, DMAR_FEADDR_REG, iommu->feaddr);
> + iommu_write_4(iommu, DMAR_FEUADDR_REG,
> + iommu->feaddr >> 32);
> + if ((iommu->flags &
> (IOMMU_FLAGS_BAD|IOMMU_FLAGS_SUSPEND)) ==
> + IOMMU_FLAGS_SUSPEND) {
> + printf("enable wakeup translation\n");
> + iommu_enable_translation(iommu, 1);
> + }
> + iommu_showcfg(iommu, -1);
> + }
> + break;
> + case DVACT_SUSPEND:
> + TAILQ_FOREACH(iommu, &sc->sc_drhds, link) {
> + printf("iommu%d suspend\n", iommu->id);
> + if (iommu->flags & IOMMU_FLAGS_BAD)
> + continue;
> + if (iommu->dte) {
> + acpiivhd_activate(iommu, act);
> + continue;
> + }
> + iommu->flags |= IOMMU_FLAGS_SUSPEND;
> + iommu_enable_translation(iommu, 0);
> + iommu_showcfg(iommu, -1);
> + }
> + break;
> + }
> + return (0);
> +}
> +
> +int
> +acpidmar_match(struct device *parent, void *match, void *aux)
> +{
> + struct acpi_attach_args *aaa = aux;
> + struct acpi_table_header *hdr;
> +
> + /* If we do not have a table, it is not us */
> + if (aaa->aaa_table == NULL)
> + return (0);
> +
> + /* If it is an DMAR table, we can attach */
> + hdr = (struct acpi_table_header *)aaa->aaa_table;
> + if (memcmp(hdr->signature, DMAR_SIG, sizeof(DMAR_SIG) - 1) == 0)
> + return (1);
> + if (memcmp(hdr->signature, IVRS_SIG, sizeof(IVRS_SIG) - 1) == 0)
> + return (1);
> +
> + return (0);
> +}
> +
> +void
> +acpidmar_attach(struct device *parent, struct device *self, void *aux)
> +{
> + struct acpidmar_softc *sc = (void *)self;
> + struct acpi_attach_args *aaa = aux;
> + struct acpi_dmar *dmar = (struct acpi_dmar *)aaa->aaa_table;
> + struct acpi_ivrs *ivrs = (struct acpi_ivrs *)aaa->aaa_table;
> + struct acpi_table_header *hdr;
> +
> + hdr = (struct acpi_table_header *)aaa->aaa_table;
> + sc->sc_memt = aaa->aaa_memt;
> + if (memcmp(hdr->signature, DMAR_SIG, sizeof(DMAR_SIG) - 1) == 0) {
> + acpidmar_sc = sc;
> + acpidmar_init(sc, dmar);
> + }
> + if (memcmp(hdr->signature, IVRS_SIG, sizeof(IVRS_SIG) - 1) == 0) {
> + acpidmar_sc = sc;
> + acpiivrs_init(sc, ivrs);
> + }
> +}
> +
> +/* Interrupt shiz */
> +void acpidmar_msi_hwmask(struct pic *, int);
> +void acpidmar_msi_hwunmask(struct pic *, int);
> +void acpidmar_msi_addroute(struct pic *, struct cpu_info *, int, int, int);
> +void acpidmar_msi_delroute(struct pic *, struct cpu_info *, int, int, int);
> +
> +void
> +acpidmar_msi_hwmask(struct pic *pic, int pin)
> +{
> + struct iommu_pic *ip = (void *)pic;
> + struct iommu_softc *iommu = ip->iommu;
> +
> + printf("msi_hwmask\n");
> +
> + mtx_enter(&iommu->reg_lock);
> +
> + iommu_write_4(iommu, DMAR_FECTL_REG, FECTL_IM);
> + iommu_read_4(iommu, DMAR_FECTL_REG);
> +
> + mtx_leave(&iommu->reg_lock);
> +}
> +
> +void
> +acpidmar_msi_hwunmask(struct pic *pic, int pin)
> +{
> + struct iommu_pic *ip = (void *)pic;
> + struct iommu_softc *iommu = ip->iommu;
> +
> + printf("msi_hwunmask\n");
> +
> + mtx_enter(&iommu->reg_lock);
> +
> + iommu_write_4(iommu, DMAR_FECTL_REG, 0);
> + iommu_read_4(iommu, DMAR_FECTL_REG);
> +
> + mtx_leave(&iommu->reg_lock);
> +}
> +
> +void
> +acpidmar_msi_addroute(struct pic *pic, struct cpu_info *ci, int pin, int vec,
> + int type)
> +{
> + struct iommu_pic *ip = (void *)pic;
> + struct iommu_softc *iommu = ip->iommu;
> +
> + mtx_enter(&iommu->reg_lock);
> +
> + iommu->fedata = vec;
> + iommu->feaddr = 0xfee00000L | (ci->ci_apicid << 12);
> + iommu_write_4(iommu, DMAR_FEDATA_REG, vec);
> + iommu_write_4(iommu, DMAR_FEADDR_REG, iommu->feaddr);
> + iommu_write_4(iommu, DMAR_FEUADDR_REG, iommu->feaddr >> 32);
> +
> + mtx_leave(&iommu->reg_lock);
> +}
> +
> +void
> +acpidmar_msi_delroute(struct pic *pic, struct cpu_info *ci, int pin, int vec,
> + int type)
> +{
> + printf("msi_delroute\n");
> +}
> +
> +void *
> +acpidmar_intr_establish(void *ctx, int level, int (*func)(void *),
> + void *arg, const char *what)
> +{
> + struct iommu_softc *iommu = ctx;
> + struct pic *pic;
> +
> + pic = &iommu->pic.pic;
> + iommu->pic.iommu = iommu;
> +
> + strlcpy(pic->pic_dev.dv_xname, "dmarpic",
> + sizeof(pic->pic_dev.dv_xname));
> + pic->pic_type = PIC_MSI;
> + pic->pic_hwmask = acpidmar_msi_hwmask;
> + pic->pic_hwunmask = acpidmar_msi_hwunmask;
> + pic->pic_addroute = acpidmar_msi_addroute;
> + pic->pic_delroute = acpidmar_msi_delroute;
> + pic->pic_edge_stubs = ioapic_edge_stubs;
> +#ifdef MULTIPROCESSOR
> + mtx_init(&pic->pic_mutex, level);
> +#endif
> +
> + return intr_establish(-1, pic, 0, IST_PULSE, level, NULL, func, arg,
> what);
> +}
> +
> +/* Intel: Handle DMAR Interrupt */
> +int
> +acpidmar_intr(void *ctx)
> +{
> + struct iommu_softc *iommu = ctx;
> + struct fault_entry fe;
> + static struct fault_entry ofe;
> + int fro, nfr, fri, i;
> + uint32_t sts;
> +
> + /*splassert(IPL_HIGH);*/
> +
> + if (!(iommu->gcmd & GCMD_TE)) {
> + return (1);
> + }
> + mtx_enter(&iommu->reg_lock);
> + sts = iommu_read_4(iommu, DMAR_FECTL_REG);
> + sts = iommu_read_4(iommu, DMAR_FSTS_REG);
> +
> + if (!(sts & FSTS_PPF)) {
> + mtx_leave(&iommu->reg_lock);
> + return (1);
> + }
> +
> + nfr = cap_nfr(iommu->cap);
> + fro = cap_fro(iommu->cap);
> + fri = (sts >> FSTS_FRI_SHIFT) & FSTS_FRI_MASK;
> + for (i = 0; i < nfr; i++) {
> + fe.hi = iommu_read_8(iommu, fro + (fri*16) + 8);
> + if (!(fe.hi & FRCD_HI_F))
> + break;
> +
> + fe.lo = iommu_read_8(iommu, fro + (fri*16));
> + if (ofe.hi != fe.hi || ofe.lo != fe.lo) {
> + iommu_showfault(iommu, fri, &fe);
> + ofe.hi = fe.hi;
> + ofe.lo = fe.lo;
> + }
> + fri = (fri + 1) % nfr;
> + }
> +
> + iommu_write_4(iommu, DMAR_FSTS_REG, FSTS_PFO | FSTS_PPF);
> +
> + mtx_leave(&iommu->reg_lock);
> +
> + return (1);
> +}
> +
> +const char *vtd_faults[] = {
> + "Software",
> + "Root Entry Not Present", /* ok (rtaddr + 4096) */
> + "Context Entry Not Present", /* ok (no CTX_P) */
> + "Context Entry Invalid", /* ok (tt = 3) */
> + "Address Beyond MGAW",
> + "Write", /* ok */
> + "Read", /* ok */
> + "Paging Entry Invalid", /* ok */
> + "Root Table Invalid",
> + "Context Table Invalid",
> + "Root Entry Reserved", /* ok (root.lo |= 0x4) */
> + "Context Entry Reserved",
> + "Paging Entry Reserved",
> + "Context Entry TT",
> + "Reserved",
> +};
> +
> +void iommu_showpte(uint64_t, int, uint64_t);
> +
> +/* Intel: Show IOMMU page table entry */
> +void
> +iommu_showpte(uint64_t ptep, int lvl, uint64_t base)
> +{
> + uint64_t nb, pb, i;
> + struct pte_entry *pte;
> +
> + pte = (void *)PMAP_DIRECT_MAP(ptep);
> + for (i = 0; i < 512; i++) {
> + if (!(pte[i].val & PTE_P))
> + continue;
> + nb = base + (i << lvl);
> + pb = pte[i].val & ~VTD_PAGE_MASK;
> + if(lvl == VTD_LEVEL0) {
> + printf(" %3llx %.16llx = %.16llx %c%c %s\n",
> + i, nb, pb,
> + pte[i].val == PTE_R ? 'r' : ' ',
> + pte[i].val & PTE_W ? 'w' : ' ',
> + (nb == pb) ? " ident" : "");
> + if (nb == pb)
> + return;
> + } else {
> + iommu_showpte(pb, lvl - VTD_STRIDE_SIZE, nb);
> + }
> + }
> +}
> +
> +/* Intel: Show IOMMU configuration */
> +void
> +iommu_showcfg(struct iommu_softc *iommu, int sid)
> +{
> + int i, j, sts, cmd;
> + struct context_entry *ctx;
> + pcitag_t tag;
> + pcireg_t clc;
> +
> + cmd = iommu_read_4(iommu, DMAR_GCMD_REG);
> + sts = iommu_read_4(iommu, DMAR_GSTS_REG);
> + printf("iommu%d: flags:%d root pa:%.16llx %s %s %s %.8x %.8x\n",
> + iommu->id, iommu->flags, iommu_read_8(iommu, DMAR_RTADDR_REG),
> + sts & GSTS_TES ? "enabled" : "disabled",
> + sts & GSTS_QIES ? "qi" : "ccmd",
> + sts & GSTS_IRES ? "ir" : "",
> + cmd, sts);
> + for (i = 0; i < 256; i++) {
> + if (!root_entry_is_valid(&iommu->root[i])) {
> + continue;
> + }
> + for (j = 0; j < 256; j++) {
> + ctx = iommu->ctx[i] + j;
> + if (!context_entry_is_valid(ctx)) {
> + continue;
> + }
> + tag = pci_make_tag(NULL, i, (j >> 3), j & 0x7);
> + clc = pci_conf_read(NULL, tag, 0x08) >> 8;
> + printf(" %.2x:%.2x.%x lvl:%d did:%.4x tt:%d
> ptep:%.16llx flag:%x cc:%.6x\n",
> + i, (j >> 3), j & 7,
> + context_address_width(ctx),
> + context_domain_id(ctx),
> + context_translation_type(ctx),
> + context_pte(ctx),
> + context_user(ctx),
> + clc);
> +#if 0
> + /* dump pagetables */
> + iommu_showpte(ctx->lo & ~VTD_PAGE_MASK, iommu->agaw -
> + VTD_STRIDE_SIZE, 0);
> +#endif
> + }
> + }
> +}
> +
> +/* Intel: Show IOMMU fault */
> +void
> +iommu_showfault(struct iommu_softc *iommu, int fri, struct fault_entry *fe)
> +{
> + int bus, dev, fun, type, fr, df;
> + bios_memmap_t *im;
> + const char *mapped;
> +
> + if (!(fe->hi & FRCD_HI_F))
> + return;
> + type = (fe->hi & FRCD_HI_T) ? 'r' : 'w';
> + fr = (fe->hi >> FRCD_HI_FR_SHIFT) & FRCD_HI_FR_MASK;
> + bus = (fe->hi >> FRCD_HI_BUS_SHIFT) & FRCD_HI_BUS_MASK;
> + dev = (fe->hi >> FRCD_HI_DEV_SHIFT) & FRCD_HI_DEV_MASK;
> + fun = (fe->hi >> FRCD_HI_FUN_SHIFT) & FRCD_HI_FUN_MASK;
> + df = (fe->hi >> FRCD_HI_FUN_SHIFT) & 0xFF;
> + iommu_showcfg(iommu, mksid(bus,dev,fun));
> + if (!iommu->ctx[bus]) {
> + /* Bus is not initialized */
> + mapped = "nobus";
> + } else if (!context_entry_is_valid(&iommu->ctx[bus][df])) {
> + /* DevFn not initialized */
> + mapped = "nodevfn";
> + } else if (context_user(&iommu->ctx[bus][df]) != 0xA) {
> + /* no bus_space_map */
> + mapped = "nomap";
> + } else {
> + /* bus_space_map */
> + mapped = "mapped";
> + }
> + printf("fri%d: dmar: %.2x:%.2x.%x %s error at %llx fr:%d [%s] iommu:%d
> [%s]\n",
> + fri, bus, dev, fun,
> + type == 'r' ? "read" : "write",
> + fe->lo,
> + fr, fr <= 13 ? vtd_faults[fr] : "unknown",
> + iommu->id,
> + mapped);
> + for (im = bios_memmap; im->type != BIOS_MAP_END; im++) {
> + if ((im->type == BIOS_MAP_RES) &&
> + (im->addr <= fe->lo) &&
> + (fe->lo <= im->addr+im->size)) {
> + printf("mem in e820.reserved\n");
> + }
> + }
> +#ifdef DDB
> + if (acpidmar_ddb)
> + db_enter();
> +#endif
> +}
> +
> diff --git a/sys/dev/acpi/acpidmar.h b/sys/dev/acpi/acpidmar.h
> new file mode 100644
> index 000000000..7938f5381
> --- /dev/null
> +++ b/sys/dev/acpi/acpidmar.h
> @@ -0,0 +1,536 @@
> +/*
> + * Copyright (c) 2015 Jordan Hargrave <jordan_hargr...@hotmail.com>
> + *
> + * Permission to use, copy, modify, and distribute this software for any
> + * purpose with or without fee is hereby granted, provided that the above
> + * copyright notice and this permission notice appear in all copies.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> + */
> +
> +#ifndef _DEV_ACPI_DMARREG_H_
> +#define _DEV_ACPI_DMARREG_H_
> +
> +/*#define IOMMU_DEBUG*/
> +
> +#define VTD_STRIDE_MASK 0x1FF
> +#define VTD_STRIDE_SIZE 9
> +#define VTD_PAGE_SIZE 4096
> +#define VTD_PAGE_MASK 0xFFF
> +#define VTD_PTE_MASK 0x0000FFFFFFFFF000LL
> +
> +#define VTD_LEVEL0 12
> +#define VTD_LEVEL1 21
> +#define VTD_LEVEL2 30 /* Minimum level supported */
> +#define VTD_LEVEL3 39 /* Also supported */
> +#define VTD_LEVEL4 48
> +#define VTD_LEVEL5 57
> +
> +#define _xbit(x,y) (((x)>> (y)) & 1)
> +#define _xfld(x,y) (uint32_t)(((x)>> y##_SHIFT) & y##_MASK)
> +
> +#define VTD_AWTOLEVEL(x) (((x) - 30) / VTD_STRIDE_SIZE)
> +#define VTD_LEVELTOAW(x) (((x) * VTD_STRIDE_SIZE) + 30)
> +
> +#define DMAR_VER_REG 0x00 /* 32:Arch version supported by this
> IOMMU */
> +#define DMAR_RTADDR_REG 0x20 /* 64:Root entry table */
> +#define DMAR_FEDATA_REG 0x3c /* 32:Fault event interrupt
> data register */
> +#define DMAR_FEADDR_REG 0x40 /* 32:Fault event interrupt
> addr register */
> +#define DMAR_FEUADDR_REG 0x44 /* 32:Upper address register */
> +#define DMAR_AFLOG_REG 0x58 /* 64:Advanced Fault control */
> +#define DMAR_PMEN_REG 0x64 /* 32:Enable Protected Memory
> Region */
> +#define DMAR_PLMBASE_REG 0x68 /* 32:PMRR Low addr */
> +#define DMAR_PLMLIMIT_REG 0x6c /* 32:PMRR low limit */
> +#define DMAR_PHMBASE_REG 0x70 /* 64:pmrr high base addr */
> +#define DMAR_PHMLIMIT_REG 0x78 /* 64:pmrr high limit */
> +#define DMAR_ICS_REG 0x9C /* 32:Invalidation complete status
> register */
> +#define DMAR_IECTL_REG 0xa0 /* 32:Invalidation event
> control register */
> +#define DMAR_IEDATA_REG 0xa4 /* 32:Invalidation event data
> register */
> +#define DMAR_IEADDR_REG 0xa8 /* 32:Invalidation event
> address register */
> +#define DMAR_IEUADDR_REG 0xac /* 32:Invalidation event upper address
> register */
> +#define DMAR_IRTA_REG 0xb8 /* 64:Interrupt remapping table
> addr register */
> +#define DMAR_CAP_REG 0x08 /* 64:Hardware supported capabilities */
> +#define CAP_PI (1LL << 59)
> +#define CAP_FL1GP (1LL << 56)
> +#define CAP_DRD (1LL << 55)
> +#define CAP_DWD (1LL << 54)
> +#define CAP_MAMV_MASK 0x3F
> +#define CAP_MAMV_SHIFT 48LL
> +#define cap_mamv(x) _xfld(x,CAP_MAMV)
> +#define CAP_NFR_MASK 0xFF
> +#define CAP_NFR_SHIFT 40LL
> +#define cap_nfr(x) (_xfld(x,CAP_NFR) + 1)
> +#define CAP_PSI (1LL << 39)
> +#define CAP_SLLPS_MASK 0xF
> +#define CAP_SLLPS_SHIFT 34LL
> +#define cap_sllps(x) _xfld(x,CAP_SLLPS)
> +#define CAP_FRO_MASK 0x3FF
> +#define CAP_FRO_SHIFT 24LL
> +#define cap_fro(x) (_xfld(x,CAP_FRO) * 16)
> +#define CAP_ZLR (1LL << 22)
> +#define CAP_MGAW_MASK 0x3F
> +#define CAP_MGAW_SHIFT 16LL
> +#define cap_mgaw(x) (_xfld(x,CAP_MGAW) + 1)
> +#define CAP_SAGAW_MASK 0x1F
> +#define CAP_SAGAW_SHIFT 8LL
> +#define cap_sagaw(x) _xfld(x,CAP_SAGAW)
> +#define CAP_CM (1LL << 7)
> +#define CAP_PHMR (1LL << 6)
> +#define CAP_PLMR (1LL << 5)
> +#define CAP_RWBF (1LL << 4)
> +#define CAP_AFL (1LL << 3)
> +#define CAP_ND_MASK 0x7
> +#define CAP_ND_SHIFT 0x00
> +#define cap_nd(x) (16 << (((x) & CAP_ND_MASK) << 1))
> +
> +#define DMAR_ECAP_REG 0x10 /* 64:Extended capabilities
> supported */
> +#define ECAP_PSS_MASK 0x1F
> +#define ECAP_PSS_SHIFT 35
> +#define ECAP_EAFS (1LL << 34)
> +#define ECAP_NWFS (1LL << 33)
> +#define ECAP_SRS (1LL << 31)
> +#define ECAP_ERS (1LL << 30)
> +#define ECAP_PRS (1LL << 29)
> +#define ECAP_PASID (1LL << 28)
> +#define ECAP_DIS (1LL << 27)
> +#define ECAP_NEST (1LL << 26)
> +#define ECAP_MTS (1LL << 25)
> +#define ECAP_ECS (1LL << 24)
> +#define ECAP_MHMV_MASK 0xF
> +#define ECAP_MHMV_SHIFT 0x20
> +#define ecap_mhmv(x) _xfld(x,ECAP_MHMV)
> +#define ECAP_IRO_MASK 0x3FF /* IOTLB Register */
> +#define ECAP_IRO_SHIFT 0x8
> +#define ecap_iro(x) (_xfld(x,ECAP_IRO) * 16)
> +#define ECAP_SC (1LL << 7) /* Snoop Control */
> +#define ECAP_PT (1LL << 6) /* HW Passthru */
> +#define ECAP_EIM (1LL << 4)
> +#define ECAP_IR (1LL << 3) /* Interrupt remap */
> +#define ECAP_DT (1LL << 2) /* Device IOTLB */
> +#define ECAP_QI (1LL << 1) /* Queued Invalidation */
> +#define ECAP_C (1LL << 0) /* Coherent cache */
> +
> +#define DMAR_GCMD_REG 0x18 /* 32:Global command
> register */
> +#define GCMD_TE (1LL << 31)
> +#define GCMD_SRTP (1LL << 30)
> +#define GCMD_SFL (1LL << 29)
> +#define GCMD_EAFL (1LL << 28)
> +#define GCMD_WBF (1LL << 27)
> +#define GCMD_QIE (1LL << 26)
> +#define GCMD_IRE (1LL << 25)
> +#define GCMD_SIRTP (1LL << 24)
> +#define GCMD_CFI (1LL << 23)
> +
> +#define DMAR_GSTS_REG 0x1c /* 32:Global status
> register */
> +#define GSTS_TES (1LL << 31)
> +#define GSTS_RTPS (1LL << 30)
> +#define GSTS_FLS (1LL << 29)
> +#define GSTS_AFLS (1LL << 28)
> +#define GSTS_WBFS (1LL << 27)
> +#define GSTS_QIES (1LL << 26)
> +#define GSTS_IRES (1LL << 25)
> +#define GSTS_IRTPS (1LL << 24)
> +#define GSTS_CFIS (1LL << 23)
> +
> +#define DMAR_CCMD_REG 0x28 /* 64:Context command
> reg */
> +#define CCMD_ICC (1LL << 63)
> +#define CCMD_CIRG_MASK 0x3
> +#define CCMD_CIRG_SHIFT 61
> +#define CCMD_CIRG(x) ((uint64_t)(x) << CCMD_CIRG_SHIFT)
> +#define CCMD_CAIG_MASK 0x3
> +#define CCMD_CAIG_SHIFT 59
> +#define CCMD_FM_MASK 0x3
> +#define CCMD_FM_SHIFT 32
> +#define CCMD_FM(x) (((uint64_t)(x) << CCMD_FM_SHIFT))
> +#define CCMD_SID_MASK 0xFFFF
> +#define CCMD_SID_SHIFT 8
> +#define CCMD_SID(x) (((x) << CCMD_SID_SHIFT))
> +#define CCMD_DID_MASK 0xFFFF
> +#define CCMD_DID_SHIFT 0
> +#define CCMD_DID(x) (((x) << CCMD_DID_SHIFT))
> +
> +#define CIG_GLOBAL CCMD_CIRG(CTX_GLOBAL)
> +#define CIG_DOMAIN CCMD_CIRG(CTX_DOMAIN)
> +#define CIG_DEVICE CCMD_CIRG(CTX_DEVICE)
> +
> +
> +#define DMAR_FSTS_REG 0x34 /* 32:Fault Status register */
> +#define FSTS_FRI_MASK 0xFF
> +#define FSTS_FRI_SHIFT 8
> +#define FSTS_PRO (1LL << 7)
> +#define FSTS_ITE (1LL << 6)
> +#define FSTS_ICE (1LL << 5)
> +#define FSTS_IQE (1LL << 4)
> +#define FSTS_APF (1LL << 3)
> +#define FSTS_APO (1LL << 2)
> +#define FSTS_PPF (1LL << 1)
> +#define FSTS_PFO (1LL << 0)
> +
> +#define DMAR_FECTL_REG 0x38 /* 32:Fault control register */
> +#define FECTL_IM (1LL << 31)
> +#define FECTL_IP (1LL << 30)
> +
> +#define FRCD_HI_F (1LL << (127-64))
> +#define FRCD_HI_T (1LL << (126-64))
> +#define FRCD_HI_AT_MASK 0x3
> +#define FRCD_HI_AT_SHIFT (124-64)
> +#define FRCD_HI_PV_MASK 0xFFFFF
> +#define FRCD_HI_PV_SHIFT (104-64)
> +#define FRCD_HI_FR_MASK 0xFF
> +#define FRCD_HI_FR_SHIFT (96-64)
> +#define FRCD_HI_PP (1LL << (95-64))
> +
> +#define FRCD_HI_SID_MASK 0xFF
> +#define FRCD_HI_SID_SHIFT 0
> +#define FRCD_HI_BUS_SHIFT 8
> +#define FRCD_HI_BUS_MASK 0xFF
> +#define FRCD_HI_DEV_SHIFT 3
> +#define FRCD_HI_DEV_MASK 0x1F
> +#define FRCD_HI_FUN_SHIFT 0
> +#define FRCD_HI_FUN_MASK 0x7
> +
> +#define DMAR_IOTLB_REG(x) (ecap_iro((x)->ecap) + 8)
> +#define DMAR_IVA_REG(x) (ecap_iro((x)->ecap) + 0)
> +
> +#define DMAR_FRIH_REG(x,i) (cap_fro((x)->cap) + 16*(i) + 8)
> +#define DMAR_FRIL_REG(x,i) (cap_fro((x)->cap) + 16*(i) + 0)
> +
> +#define IOTLB_IVT (1LL << 63)
> +#define IOTLB_IIRG_MASK 0x3
> +#define IOTLB_IIRG_SHIFT 60
> +#define IOTLB_IIRG(x) ((uint64_t)(x) << IOTLB_IIRG_SHIFT)
> +#define IOTLB_IAIG_MASK 0x3
> +#define IOTLB_IAIG_SHIFT 57
> +#define IOTLB_DR (1LL << 49)
> +#define IOTLB_DW (1LL << 48)
> +#define IOTLB_DID_MASK 0xFFFF
> +#define IOTLB_DID_SHIFT 32
> +#define IOTLB_DID(x) ((uint64_t)(x) << IOTLB_DID_SHIFT)
> +
> +#define IIG_GLOBAL IOTLB_IIRG(IOTLB_GLOBAL)
> +#define IIG_DOMAIN IOTLB_IIRG(IOTLB_DOMAIN)
> +#define IIG_PAGE IOTLB_IIRG(IOTLB_PAGE)
> +
> +#define DMAR_IQH_REG 0x80 /* 64:Invalidation queue head register */
> +#define DMAR_IQT_REG 0x88 /* 64:Invalidation queue tail register */
> +#define DMAR_IQA_REG 0x90 /* 64:Invalidation queue addr register */
> +#define IQA_QS_256 0 /* 256 entries */
> +#define IQA_QS_512 1 /* 512 */
> +#define IQA_QS_1K 2 /* 1024 */
> +#define IQA_QS_2K 3 /* 2048 */
> +#define IQA_QS_4K 4 /* 4096 */
> +#define IQA_QS_8K 5 /* 8192 */
> +#define IQA_QS_16K 6 /* 16384 */
> +#define IQA_QS_32K 7 /* 32768 */
> +
> +/* Read-Modify-Write helpers */
> +static inline void iommu_rmw32(void *ov, uint32_t mask, uint32_t shift,
> uint32_t nv)
> +{
> + *(uint32_t *)ov &= ~(mask << shift);
> + *(uint32_t *)ov |= (nv & mask) << shift;
> +}
> +static inline void iommu_rmw64(void *ov, uint32_t mask, uint32_t shift,
> uint64_t nv)
> +{
> + *(uint64_t *)ov &= ~(mask << shift);
> + *(uint64_t *)ov |= (nv & mask) << shift;
> +}
> +
> +/*
> + * Root Entry: one per bus (256 x 128 bit = 4k)
> + * 0 = Present
> + * 1:11 = Reserved
> + * 12:HAW-1 = Context Table Pointer
> + * HAW:63 = Reserved
> + * 64:127 = Reserved
> + */
> +#define ROOT_P (1L << 0)
> +struct root_entry {
> + uint64_t lo;
> + uint64_t hi;
> +};
> +
> +/* Check if root entry is valid */
> +static inline bool
> +root_entry_is_valid(struct root_entry *re)
> +{
> + return (re->lo & ROOT_P);
> +}
> +
> +/*
> + * Context Entry: one per devfn (256 x 128 bit = 4k)
> + * 0 = Present
> + * 1 = Fault Processing Disable
> + * 2:3 = Translation Type
> + * 4:11 = Reserved
> + * 12:63 = Second Level Page Translation
> + * 64:66 = Address Width (# PTE levels)
> + * 67:70 = Ignore
> + * 71 = Reserved
> + * 72:87 = Domain ID
> + * 88:127 = Reserved
> + */
> +#define CTX_P (1L << 0)
> +#define CTX_FPD (1L << 1)
> +#define CTX_T_MASK 0x3
> +#define CTX_T_SHIFT 2
> +enum {
> + CTX_T_MULTI,
> + CTX_T_IOTLB,
> + CTX_T_PASSTHRU
> +};
> +
> +#define CTX_H_AW_MASK 0x7
> +#define CTX_H_AW_SHIFT 0
> +#define CTX_H_USER_MASK 0xF
> +#define CTX_H_USER_SHIFT 3
> +#define CTX_H_DID_MASK 0xFFFF
> +#define CTX_H_DID_SHIFT 8
> +
> +struct context_entry {
> + uint64_t lo;
> + uint64_t hi;
> +};
> +
> +/* Set fault processing enable/disable */
> +static inline void
> +context_set_fpd(struct context_entry *ce, int enable)
> +{
> + ce->lo &= ~CTX_FPD;
> + if (enable)
> + ce->lo |= CTX_FPD;
> +}
> +
> +/* Set context entry present */
> +static inline void
> +context_set_present(struct context_entry *ce)
> +{
> + ce->lo |= CTX_P;
> +}
> +
> +/* Set Second Level Page Table Entry PA */
> +static inline void
> +context_set_slpte(struct context_entry *ce, paddr_t slpte)
> +{
> + ce->lo &= VTD_PAGE_MASK;
> + ce->lo |= (slpte & ~VTD_PAGE_MASK);
> +}
> +
> +/* Set translation type */
> +static inline void
> +context_set_translation_type(struct context_entry *ce, int tt)
> +{
> + ce->lo &= ~(CTX_T_MASK << CTX_T_SHIFT);
> + ce->lo |= ((tt & CTX_T_MASK) << CTX_T_SHIFT);
> +}
> +
> +/* Set Address Width (# of Page Table levels) */
> +static inline void
> +context_set_address_width(struct context_entry *ce, int lvl)
> +{
> + ce->hi &= ~(CTX_H_AW_MASK << CTX_H_AW_SHIFT);
> + ce->hi |= ((lvl & CTX_H_AW_MASK) << CTX_H_AW_SHIFT);
> +}
> +
> +/* Set domain ID */
> +static inline void
> +context_set_domain_id(struct context_entry *ce, int did)
> +{
> + ce->hi &= ~(CTX_H_DID_MASK << CTX_H_DID_SHIFT);
> + ce->hi |= ((did & CTX_H_DID_MASK) << CTX_H_DID_SHIFT);
> +}
> +
> +/* Get Second Level Page Table PA */
> +static inline uint64_t
> +context_pte(struct context_entry *ce)
> +{
> + return (ce->lo & ~VTD_PAGE_MASK);
> +}
> +
> +/* Get translation type */
> +static inline int
> +context_translation_type(struct context_entry *ce)
> +{
> + return (ce->lo >> CTX_T_SHIFT) & CTX_T_MASK;
> +}
> +
> +/* Get domain ID */
> +static inline int
> +context_domain_id(struct context_entry *ce)
> +{
> + return (ce->hi >> CTX_H_DID_SHIFT) & CTX_H_DID_MASK;
> +}
> +
> +/* Get Address Width */
> +static inline int
> +context_address_width(struct context_entry *ce)
> +{
> + return VTD_LEVELTOAW((ce->hi >> CTX_H_AW_SHIFT) & CTX_H_AW_MASK);
> +}
> +
> +/* Check if context entry is valid */
> +static inline bool
> +context_entry_is_valid(struct context_entry *ce)
> +{
> + return (ce->lo & CTX_P);
> +}
> +
> +/* User-available bits in context entry */
> +static inline int
> +context_user(struct context_entry *ce)
> +{
> + return (ce->hi >> CTX_H_USER_SHIFT) & CTX_H_USER_MASK;
> +}
> +
> +static inline void
> +context_set_user(struct context_entry *ce, int v)
> +{
> + ce->hi &= ~(CTX_H_USER_MASK << CTX_H_USER_SHIFT);
> + ce->hi |= ((v & CTX_H_USER_MASK) << CTX_H_USER_SHIFT);
> +}
> +
> +/*
> + * Fault entry
> + * 0..HAW-1 = Fault address
> + * HAW:63 = Reserved
> + * 64:71 = Source ID
> + * 96:103 = Fault Reason
> + * 104:123 = PV
> + * 124:125 = Address Translation type
> + * 126 = Type (0 = Read, 1 = Write)
> + * 127 = Fault bit
> + */
> +struct fault_entry
> +{
> + uint64_t lo;
> + uint64_t hi;
> +};
> +
> +/* PTE Entry: 512 x 64-bit = 4k */
> +#define PTE_P (1L << 0)
> +#define PTE_R 0x00
> +#define PTE_W (1L << 1)
> +#define PTE_US (1L << 2)
> +#define PTE_PWT (1L << 3)
> +#define PTE_PCD (1L << 4)
> +#define PTE_A (1L << 5)
> +#define PTE_D (1L << 6)
> +#define PTE_PAT (1L << 7)
> +#define PTE_G (1L << 8)
> +#define PTE_EA (1L << 10)
> +#define PTE_XD (1LL << 63)
> +
> +/* PDE Level entry */
> +#define PTE_PS (1L << 7)
> +
> +/* PDPE Level entry */
> +
> +/* ----------------------------------------------------------------
> + * 5555555444444444333333333222222222111111111000000000------------
> + * [PML4 ->] PDPE.1GB
> + * [PML4 ->] PDPE.PDE -> PDE.2MB
> + * [PML4 ->] PDPE.PDE -> PDE -> PTE
> + * GAW0 = (12.20) (PTE)
> + * GAW1 = (21.29) (PDE)
> + * GAW2 = (30.38) (PDPE)
> + * GAW3 = (39.47) (PML4)
> + * GAW4 = (48.57) (n/a)
> + * GAW5 = (58.63) (n/a)
> + */
> +struct pte_entry {
> + uint64_t val;
> +};
> +
> +/*
> + * Queued Invalidation entry
> + * 0:3 = 01h
> + * 4:5 = Granularity
> + * 6:15 = Reserved
> + * 16:31 = Domain ID
> + * 32:47 = Source ID
> + * 48:49 = FM
> + */
> +
> +/* Invalidate Context Entry */
> +#define QI_CTX_DID_MASK 0xFFFF
> +#define QI_CTX_DID_SHIFT 16
> +#define QI_CTX_SID_MASK 0xFFFF
> +#define QI_CTX_SID_SHIFT 32
> +#define QI_CTX_FM_MASK 0x3
> +#define QI_CTX_FM_SHIFT 48
> +#define QI_CTX_IG_MASK 0x3
> +#define QI_CTX_IG_SHIFT 4
> +#define QI_CTX_DID(x) (((uint64_t)(x) << QI_CTX_DID_SHIFT))
> +#define QI_CTX_SID(x) (((uint64_t)(x) << QI_CTX_SID_SHIFT))
> +#define QI_CTX_FM(x) (((uint64_t)(x) << QI_CTX_FM_SHIFT))
> +
> +#define QI_CTX_IG_GLOBAL (CTX_GLOBAL << QI_CTX_IG_SHIFT)
> +#define QI_CTX_IG_DOMAIN (CTX_DOMAIN << QI_CTX_IG_SHIFT)
> +#define QI_CTX_IG_DEVICE (CTX_DEVICE << QI_CTX_IG_SHIFT)
> +
> +/* Invalidate IOTLB Entry */
> +#define QI_IOTLB_DID_MASK 0xFFFF
> +#define QI_IOTLB_DID_SHIFT 16
> +#define QI_IOTLB_IG_MASK 0x3
> +#define QI_IOTLB_IG_SHIFT 4
> +#define QI_IOTLB_DR (1LL << 6)
> +#define QI_IOTLB_DW (1LL << 5)
> +#define QI_IOTLB_DID(x) (((uint64_t)(x) << QI_IOTLB_DID_SHIFT))
> +
> +#define QI_IOTLB_IG_GLOBAL (1 << QI_IOTLB_IG_SHIFT)
> +#define QI_IOTLB_IG_DOMAIN (2 << QI_IOTLB_IG_SHIFT)
> +#define QI_IOTLB_IG_PAGE (3 << QI_IOTLB_IG_SHIFT)
> +
> +/* QI Commands */
> +#define QI_CTX 0x1
> +#define QI_IOTLB 0x2
> +#define QI_DEVTLB 0x3
> +#define QI_INTR 0x4
> +#define QI_WAIT 0x5
> +#define QI_EXTTLB 0x6
> +#define QI_PAS 0x7
> +#define QI_EXTDEV 0x8
> +
> +struct qi_entry {
> + uint64_t lo;
> + uint64_t hi;
> +};
> +
> +enum {
> + CTX_GLOBAL = 1,
> + CTX_DOMAIN,
> + CTX_DEVICE,
> +
> + IOTLB_GLOBAL = 1,
> + IOTLB_DOMAIN,
> + IOTLB_PAGE,
> +};
> +
> +enum {
> + VTD_FAULT_ROOT_P = 0x1, /* P field in root entry is 0 */
> + VTD_FAULT_CTX_P = 0x2, /* P field in context entry is 0 */
> + VTD_FAULT_CTX_INVAL = 0x3, /* context AW/TT/SLPPTR invalid */
> + VTD_FAULT_LIMIT = 0x4, /* Address is outside of MGAW */
> + VTD_FAULT_WRITE = 0x5, /* Address-translation fault,
> non-writable */
> + VTD_FAULT_READ = 0x6, /* Address-translation fault,
> non-readable */
> + VTD_FAULT_PTE_INVAL = 0x7, /* page table hw access error */
> + VTD_FAULT_ROOT_INVAL = 0x8, /* root table hw access error */
> + VTD_FAULT_CTX_TBL_INVAL = 0x9, /* context entry hw access error */
> + VTD_FAULT_ROOT_RESERVED = 0xa, /* non-zero reserved field in root
> entry */
> + VTD_FAULT_CTX_RESERVED = 0xb, /* non-zero reserved field in context
> entry */
> + VTD_FAULT_PTE_RESERVED = 0xc, /* non-zero reserved field in paging
> entry */
> + VTD_FAULT_CTX_TT = 0xd, /* invalid translation type */
> +};
> +
> +#endif
> +
> +void acpidmar_pci_hook(pci_chipset_tag_t, struct pci_attach_args *);
> +void dmar_ptmap(bus_dma_tag_t, bus_addr_t);
> +void acpidmar_sw(int);
> +
> +#define __EXTRACT(v,m) (((v) >> m##_SHIFT) & m##_MASK)
> diff --git a/sys/dev/acpi/amd_iommu.h b/sys/dev/acpi/amd_iommu.h
> new file mode 100644
> index 000000000..30de2593e
> --- /dev/null
> +++ b/sys/dev/acpi/amd_iommu.h
> @@ -0,0 +1,360 @@
> +/*
> + * Copyright (c) 2019 Jordan Hargrave <jordan_hargr...@hotmail.com>
> + *
> + * Permission to use, copy, modify, and distribute this software for any
> + * purpose with or without fee is hereby granted, provided that the above
> + * copyright notice and this permission notice appear in all copies.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> + */
> +#ifndef __amd_iommu_h__
> +#define __amd_iommu_h__
> +
> +#define DEV_TAB_BASE_REG 0x0000
> +#define CMD_BASE_REG 0x0008
> +#define EVT_BASE_REG 0x0010
> +
> +#define EXCL_BASE_REG 0x0020
> +#define EXCL_LIMIT_REG 0x0028
> +
> +/* Extended Feature Register */
> +#define EXTFEAT_REG 0x0030
> +#define EFR_PREFSUP (1L << 0)
> +#define EFR_PPRSUP (1L << 1)
> +#define EFR_NXSUP (1L << 3)
> +#define EFR_GTSUP (1L << 4)
> +#define EFR_IASUP (1L << 6)
> +#define EFR_GASUP (1L << 7)
> +#define EFR_HESUP (1L << 8)
> +#define EFR_PCSUP (1L << 9)
> +#define EFR_HATS_SHIFT 10
> +#define EFR_HATS_MASK 0x3
> +#define EFR_GATS_SHIFT 12
> +#define EFR_GATS_MASK 0x3
> +#define EFR_GLXSUP_SHIFT 14
> +#define EFR_GLXSUP_MASK 0x3
> +#define EFR_SMIFSUP_SHIFT 16
> +#define EFR_SMIFSUP_MASK 0x3
> +#define EFR_SMIFRC_SHIFT 18
> +#define EFR_SMIFRC_MASK 0x7
> +#define EFR_GAMSUP_SHIFT 21
> +#define EFR_GAMSUP_MASK 0x7
> +
> +#define CMD_HEAD_REG 0x2000
> +#define CMD_TAIL_REG 0x2008
> +#define EVT_HEAD_REG 0x2010
> +#define EVT_TAIL_REG 0x2018
> +
> +#define IOMMUSTS_REG 0x2020
> +
> +#define DEV_TAB_MASK 0x000FFFFFFFFFF000LL
> +#define DEV_TAB_LEN 0x1FF
> +
> +/* IOMMU Control */
> +#define IOMMUCTL_REG 0x0018
> +#define CTL_IOMMUEN (1L << 0)
> +#define CTL_HTTUNEN (1L << 1)
> +#define CTL_EVENTLOGEN (1L << 2)
> +#define CTL_EVENTINTEN (1L << 3)
> +#define CTL_COMWAITINTEN (1L << 4)
> +#define CTL_INVTIMEOUT_SHIFT 5
> +#define CTL_INVTIMEOUT_MASK 0x7
> +#define CTL_INVTIMEOUT_NONE 0
> +#define CTL_INVTIMEOUT_1MS 1
> +#define CTL_INVTIMEOUT_10MS 2
> +#define CTL_INVTIMEOUT_100MS 3
> +#define CTL_INVTIMEOUT_1S 4
> +#define CTL_INVTIMEOUT_10S 5
> +#define CTL_INVTIMEOUT_100S 6
> +#define CTL_PASSPW (1L << 8)
> +#define CTL_RESPASSPW (1L << 9)
> +#define CTL_COHERENT (1L << 10)
> +#define CTL_ISOC (1L << 11)
> +#define CTL_CMDBUFEN (1L << 12)
> +#define CTL_PPRLOGEN (1L << 13)
> +#define CTL_PPRINTEN (1L << 14)
> +#define CTL_PPREN (1L << 15)
> +#define CTL_GTEN (1L << 16)
> +#define CTL_GAEN (1L << 17)
> +#define CTL_CRW_SHIFT 18
> +#define CTL_CRW_MASK 0xF
> +#define CTL_SMIFEN (1L << 22)
> +#define CTL_SLFWBDIS (1L << 23)
> +#define CTL_SMIFLOGEN (1L << 24)
> +#define CTL_GAMEN_SHIFT 25
> +#define CTL_GAMEN_MASK 0x7
> +#define CTL_GALOGEN (1L << 28)
> +#define CTL_GAINTEN (1L << 29)
> +#define CTL_DUALPPRLOGEN_SHIFT 30
> +#define CTL_DUALPPRLOGEN_MASK 0x3
> +#define CTL_DUALEVTLOGEN_SHIFT 32
> +#define CTL_DUALEVTLOGEN_MASK 0x3
> +#define CTL_DEVTBLSEGEN_SHIFT 34
> +#define CTL_DEVTBLSEGEN_MASK 0x7
> +#define CTL_PRIVABRTEN_SHIFT 37
> +#define CTL_PRIVABRTEN_MASK 0x3
> +#define CTL_PPRAUTORSPEN (1LL << 39)
> +#define CTL_MARCEN (1LL << 40)
> +#define CTL_BLKSTOPMRKEN (1LL << 41)
> +#define CTL_PPRAUTOSPAON (1LL << 42)
> +#define CTL_DOMAINIDPNE (1LL << 43)
> +
> +#define CMD_BASE_MASK 0x000FFFFFFFFFF000LL
> +#define CMD_TBL_SIZE 4096
> +#define CMD_TBL_LEN_4K (8LL << 56)
> +#define CMD_TBL_LEN_8K (9lL << 56)
> +
> +#define EVT_BASE_MASK 0x000FFFFFFFFFF000LL
> +#define EVT_TBL_SIZE 4096
> +#define EVT_TBL_LEN_4K (8LL << 56)
> +#define EVT_TBL_LEN_8K (9LL << 56)
> +
> +/*========================
> + * DEVICE TABLE ENTRY
> + * Contains mapping of bus-device-function
> + *
> + * 0 Valid (V)
> + * 1 Translation Valid (TV)
> + * 7:8 Host Address Dirty (HAD)
> + * 9:11 Page Table Depth (usually 4)
> + * 12:51 Page Table Physical Address
> + * 52 PPR Enable
> + * 53 GPRP
> + * 54 Guest I/O Protection Valid (GIoV)
> + * 55 Guest Translation Valid (GV)
> + * 56:57 Guest Levels translated (GLX)
> + * 58:60 Guest CR3 bits 12:14 (GCR3TRP)
> + * 61 I/O Read Permission (IR)
> + * 62 I/O Write Permission (IW)
> + * 64:79 Domain ID
> + * 80:95 Guest CR3 bits 15:30 (GCR3TRP)
> + * 96 IOTLB Enable (I)
> + * 97 Suppress multiple I/O page faults (I)
> + * 98 Supress all I/O page faults (SA)
> + * 99:100 Port I/O Control (IoCTL)
> + * 101 Cache IOTLB Hint
> + * 102 Snoop Disable (SD)
> + * 103 Allow Exclusion (EX)
> + * 104:105 System Management Message (SysMgt)
> + * 107:127 Guest CR3 bits 31:51 (GCR3TRP)
> + * 128 Interrupt Map Valid (IV)
> + * 129:132 Interrupt Table Length (IntTabLen)
> + *========================*/
> +struct ivhd_dte {
> + uint32_t dw0;
> + uint32_t dw1;
> + uint32_t dw2;
> + uint32_t dw3;
> + uint32_t dw4;
> + uint32_t dw5;
> + uint32_t dw6;
> + uint32_t dw7;
> +} __packed;
> +
> +#define HWDTE_SIZE (65536 * sizeof(struct ivhd_dte))
> +
> +#define DTE_V (1L << 0) /* dw0
> */
> +#define DTE_TV (1L << 1) /* dw0
> */
> +#define DTE_LEVEL_SHIFT 9 /* dw0
> */
> +#define DTE_LEVEL_MASK 0x7 /* dw0
> */
> +#define DTE_HPTRP_MASK 0x000FFFFFFFFFF000LL /*
> dw0,1 */
> +
> +#define DTE_PPR (1L << 20) /* dw1
> */
> +#define DTE_GPRP (1L << 21) /* dw1 */
> +#define DTE_GIOV (1L << 22) /* dw1 */
> +#define DTE_GV (1L << 23) /* dw1
> */
> +#define DTE_IR (1L << 29) /* dw1
> */
> +#define DTE_IW (1L << 30) /* dw1
> */
> +
> +#define DTE_DID_MASK 0xFFFF /* dw2 */
> +
> +#define DTE_IV (1L << 0) /* dw3
> */
> +#define DTE_SE (1L << 1)
> +#define DTE_SA (1L << 2)
> +#define DTE_INTTABLEN_SHIFT 1
> +#define DTE_INTTABLEN_MASK 0xF
> +#define DTE_IRTP_MASK 0x000FFFFFFFFFFFC0LL
> +
> +#define PTE_LVL5 48
> +#define PTE_LVL4 39
> +#define PTE_LVL3 30
> +#define PTE_LVL2 21
> +#define PTE_LVL1 12
> +
> +#define PTE_NXTLVL(x) (((x) & 0x7) << 9)
> +#define PTE_PADDR_MASK 0x000FFFFFFFFFF000LL
> +#define PTE_IR (1LL << 61)
> +#define PTE_IW (1LL << 62)
> +
> +#define DTE_GCR312_MASK 0x3
> +#define DTE_GCR312_SHIFT 24
> +
> +#define DTE_GCR315_MASK 0xFFFF
> +#define DTE_GCR315_SHIFT 16
> +
> +#define DTE_GCR331_MASK 0xFFFFF
> +#define DTE_GCR331_SHIFT 12
> +
> +#define _get64(x) *(uint64_t *)(x)
> +#define _put64(x,v) *(uint64_t *)(x) = (v)
> +
> +/* Set Guest CR3 address */
> +static inline void
> +dte_set_guest_cr3(struct ivhd_dte *dte, paddr_t paddr)
> +{
> + iommu_rmw32(&dte->dw1, DTE_GCR312_MASK, DTE_GCR312_SHIFT, paddr >> 12);
> + iommu_rmw32(&dte->dw2, DTE_GCR315_MASK, DTE_GCR315_SHIFT, paddr >> 15);
> + iommu_rmw32(&dte->dw3, DTE_GCR331_MASK, DTE_GCR331_SHIFT, paddr >> 31);
> +}
> +
> +/* Set Interrupt Remapping Root Pointer */
> +static inline void
> +dte_set_interrupt_table_root_ptr(struct ivhd_dte *dte, paddr_t paddr)
> +{
> + uint64_t ov = _get64(&dte->dw4);
> + _put64(&dte->dw4, (ov & ~DTE_IRTP_MASK) | (paddr & DTE_IRTP_MASK));
> +}
> +
> +/* Set Interrupt Remapping Table length */
> +static inline void
> +dte_set_interrupt_table_length(struct ivhd_dte *dte, int nEnt)
> +{
> + iommu_rmw32(&dte->dw4, DTE_INTTABLEN_MASK, DTE_INTTABLEN_SHIFT, nEnt);
> +}
> +
> +/* Set Interrupt Remapping Valid */
> +static inline void
> +dte_set_interrupt_valid(struct ivhd_dte *dte)
> +{
> + dte->dw4 |= DTE_IV;
> +}
> +
> +/* Set Domain ID in Device Table Entry */
> +static inline void
> +dte_set_domain(struct ivhd_dte *dte, uint16_t did)
> +{
> + dte->dw2 = (dte->dw2 & ~DTE_DID_MASK) | (did & DTE_DID_MASK);
> +}
> +
> +/* Set Page Table Pointer for device */
> +static inline void
> +dte_set_host_page_table_root_ptr(struct ivhd_dte *dte, paddr_t paddr)
> +{
> + uint64_t ov;
> +
> + ov = _get64(&dte->dw0) & ~DTE_HPTRP_MASK;
> + ov |= (paddr & DTE_HPTRP_MASK) | PTE_IW | PTE_IR;
> +
> + _put64(&dte->dw0, ov);
> +}
> +
> +/* Set Page Table Levels Mask */
> +static inline void
> +dte_set_mode(struct ivhd_dte *dte, int mode)
> +{
> + iommu_rmw32(&dte->dw0, DTE_LEVEL_MASK, DTE_LEVEL_SHIFT, mode);
> +}
> +
> +static inline void
> +dte_set_tv(struct ivhd_dte *dte)
> +{
> + dte->dw0 |= DTE_TV;
> +}
> +
> +/* Set Device Table Entry valid.
> + * Domain/Level/Mode/PageTable should already be set
> + */
> +static inline void
> +dte_set_valid(struct ivhd_dte *dte)
> +{
> + dte->dw0 |= DTE_V;
> +}
> +
> +/* Check if Device Table Entry is valid */
> +static inline int
> +dte_is_valid(struct ivhd_dte *dte)
> +{
> + return (dte->dw0 & DTE_V);
> +}
> +
> +/*=========================================
> + * COMMAND
> + *=========================================*/
> +struct ivhd_command {
> + uint32_t dw0;
> + uint32_t dw1;
> + uint32_t dw2;
> + uint32_t dw3;
> +} __packed;
> +
> +#define CMD_SHIFT 28
> +
> +enum {
> + COMPLETION_WAIT = 0x01,
> + INVALIDATE_DEVTAB_ENTRY = 0x02,
> + INVALIDATE_IOMMU_PAGES = 0x03,
> + INVALIDATE_IOTLB_PAGES = 0x04,
> + INVALIDATE_INTERRUPT_TABLE = 0x05,
> + PREFETCH_IOMMU_PAGES = 0x06,
> + COMPLETE_PPR_REQUEST = 0x07,
> + INVALIDATE_IOMMU_ALL = 0x08,
> +};
> +
> +/*=========================================
> + * EVENT
> + *=========================================*/
> +struct ivhd_event {
> + uint32_t dw0;
> + uint32_t dw1;
> + uint32_t dw2;
> + uint32_t dw3;
> +} __packed;
> +
> +#define EVT_TYPE_SHIFT 28
> +#define EVT_TYPE_MASK 0xF
> +#define EVT_SID_SHIFT 0
> +#define EVT_SID_MASK 0xFFFF
> +#define EVT_DID_SHIFT 0
> +#define EVT_DID_MASK 0xFFFF
> +#define EVT_FLAG_SHIFT 16
> +#define EVT_FLAG_MASK 0xFFF
> +
> +/* IOMMU Fault reasons */
> +enum {
> + ILLEGAL_DEV_TABLE_ENTRY = 0x1,
> + IO_PAGE_FAULT = 0x2,
> + DEV_TAB_HARDWARE_ERROR = 0x3,
> + PAGE_TAB_HARDWARE_ERROR = 0x4,
> + ILLEGAL_COMMAND_ERROR = 0x5,
> + COMMAND_HARDWARE_ERROR = 0x6,
> + IOTLB_INV_TIMEOUT = 0x7,
> + INVALID_DEVICE_REQUEST = 0x8,
> +};
> +
> +#define EVT_GN (1L << 16)
> +#define EVT_NX (1L << 17)
> +#define EVT_US (1L << 18)
> +#define EVT_I (1L << 19)
> +#define EVT_PR (1L << 20)
> +#define EVT_RW (1L << 21)
> +#define EVT_PE (1L << 22)
> +#define EVT_RZ (1L << 23)
> +#define EVT_TR (1L << 24)
> +
> +struct iommu_softc;
> +
> +int ivhd_flush_devtab(struct iommu_softc *, int);
> +int ivhd_invalidate_iommu_all(struct iommu_softc *);
> +int ivhd_invalidate_interrupt_table(struct iommu_softc *, int);
> +int ivhd_issue_command(struct iommu_softc *, const struct ivhd_command *,
> int);
> +int ivhd_invalidate_domain(struct iommu_softc *, int);
> +
> +void _dumppte(struct pte_entry *, int, vaddr_t);
> +
> +#endif
> diff --git a/sys/dev/acpi/files.acpi b/sys/dev/acpi/files.acpi
> index 252cc5bcf..13bcce896 100644
> --- a/sys/dev/acpi/files.acpi
> +++ b/sys/dev/acpi/files.acpi
> @@ -70,6 +70,11 @@ device acpiprt
> attach acpiprt at acpi
> file dev/acpi/acpiprt.c acpiprt needs-flag
>
> +# DMAR device
> +device acpidmar
> +attach acpidmar at acpi
> +file dev/acpi/acpidmar.c acpidmar needs-flag
> +
> # Docking station
> device acpidock
> attach acpidock at acpi
>
