This is the first patch for adding PCI passthrough support to VMD.
I am splitting up the necessary changes into smaller patches.
This code fixes the pci device union for accessing PCI config space >= 0x40
pcidump -xxx would return garbage data due to union overlap
pci_add_bar now requires specifying the BAR area size to allocate
Extra cookie argument added to pci_add_device
---
usr.sbin/vmd/pci.c | 50 +++++++++++++++++++--------
usr.sbin/vmd/pci.h | 80 ++++++++++++++++++++++---------------------
usr.sbin/vmd/virtio.c | 25 +++++++-------
3 files changed, 89 insertions(+), 66 deletions(-)
diff --git a/usr.sbin/vmd/pci.c b/usr.sbin/vmd/pci.c
index 954235eb6..4ce393237 100644
--- a/usr.sbin/vmd/pci.c
+++ b/usr.sbin/vmd/pci.c
@@ -39,28 +39,47 @@ extern char *__progname;
const uint8_t pci_pic_irqs[PCI_MAX_PIC_IRQS] = {3, 5, 6, 7, 9, 10, 11, 12,
14, 15};
+/*
+ * pci_mkbar
+ *
+ * Calculates BAR address given a size and alignment.
+ * Returns allocated address and updates next address
+ * Returns zero if address is out of range
+ */
+static uint64_t
+pci_mkbar(uint64_t *base, uint32_t size, uint32_t align, uint64_t maxbase)
+{
+ uint64_t mask = size - 1;
+ uint64_t cbase;
+
+ if (*base >= maxbase)
+ return (0);
+ cbase = *base;
+ *base = (*base + size + mask) & ~mask;
+ return cbase;
+}
+
/*
* pci_add_bar
*
* Adds a BAR for the PCI device 'id'. On access, 'barfn' will be
* called, and passed 'cookie' as an identifier.
*
- * BARs are fixed size, meaning all I/O BARs requested have the
- * same size and all MMIO BARs have the same size.
- *
* Parameters:
* id: PCI device to add the BAR to (local count, eg if id == 4,
* this BAR is to be added to the VM's 5th PCI device)
* type: type of the BAR to add (PCI_MAPREG_TYPE_xxx)
+ * size: Size of BAR area
* barfn: callback function invoked on BAR access
* cookie: cookie passed to barfn on access
*
* Returns 0 if the BAR was added successfully, 1 otherwise.
*/
int
-pci_add_bar(uint8_t id, uint32_t type, void *barfn, void *cookie)
+pci_add_bar(uint8_t id, uint32_t type, uint32_t size, void *barfn, void
*cookie)
{
uint8_t bar_reg_idx, bar_ct;
+ uint64_t base = 0;
/* Check id */
if (id >= pci.pci_dev_ct)
@@ -74,31 +93,31 @@ pci_add_bar(uint8_t id, uint32_t type, void *barfn, void
*cookie)
/* Compute BAR address and add */
bar_reg_idx = (PCI_MAPREG_START + (bar_ct * 4)) / 4;
if (type == PCI_MAPREG_TYPE_MEM) {
- if (pci.pci_next_mmio_bar >= VMM_PCI_MMIO_BAR_END)
+ base = pci_mkbar(&pci.pci_next_mmio_bar, size, 4096,
VMM_PCI_MMIO_BAR_END);
+ if (base == 0)
return (1);
pci.pci_devices[id].pd_cfg_space[bar_reg_idx] =
- PCI_MAPREG_MEM_ADDR(pci.pci_next_mmio_bar);
- pci.pci_next_mmio_bar += VMM_PCI_MMIO_BAR_SIZE;
+ PCI_MAPREG_MEM_ADDR(base);
pci.pci_devices[id].pd_barfunc[bar_ct] = barfn;
pci.pci_devices[id].pd_bar_cookie[bar_ct] = cookie;
pci.pci_devices[id].pd_bartype[bar_ct] = PCI_BAR_TYPE_MMIO;
- pci.pci_devices[id].pd_barsize[bar_ct] = VMM_PCI_MMIO_BAR_SIZE;
+ pci.pci_devices[id].pd_barsize[bar_ct] = size;
pci.pci_devices[id].pd_bar_ct++;
} else if (type == PCI_MAPREG_TYPE_IO) {
- if (pci.pci_next_io_bar >= VMM_PCI_IO_BAR_END)
+ base = pci_mkbar(&pci.pci_next_io_bar, size, 4,
VMM_PCI_IO_BAR_END);
+ if (base == 0)
return (1);
pci.pci_devices[id].pd_cfg_space[bar_reg_idx] =
- PCI_MAPREG_IO_ADDR(pci.pci_next_io_bar) |
+ PCI_MAPREG_IO_ADDR(base) |
PCI_MAPREG_TYPE_IO;
- pci.pci_next_io_bar += VMM_PCI_IO_BAR_SIZE;
pci.pci_devices[id].pd_barfunc[bar_ct] = barfn;
pci.pci_devices[id].pd_bar_cookie[bar_ct] = cookie;
DPRINTF("%s: adding pci bar cookie for dev %d bar %d = %p",
__progname, id, bar_ct, cookie);
pci.pci_devices[id].pd_bartype[bar_ct] = PCI_BAR_TYPE_IO;
- pci.pci_devices[id].pd_barsize[bar_ct] = VMM_PCI_IO_BAR_SIZE;
+ pci.pci_devices[id].pd_barsize[bar_ct] = size;
pci.pci_devices[id].pd_bar_ct++;
}
@@ -165,7 +184,7 @@ pci_get_dev_irq(uint8_t id)
int
pci_add_device(uint8_t *id, uint16_t vid, uint16_t pid, uint8_t class,
uint8_t subclass, uint16_t subsys_vid, uint16_t subsys_id,
- uint8_t irq_needed, pci_cs_fn_t csfunc)
+ uint8_t irq_needed, pci_cs_fn_t csfunc, void *cookie)
{
/* Exceeded max devices? */
if (pci.pci_dev_ct >= PCI_CONFIG_MAX_DEV)
@@ -186,6 +205,7 @@ pci_add_device(uint8_t *id, uint16_t vid, uint16_t pid,
uint8_t class,
pci.pci_devices[*id].pd_subsys_id = subsys_id;
pci.pci_devices[*id].pd_csfunc = csfunc;
+ pci.pci_devices[*id].pd_cookie = cookie;
if (irq_needed) {
pci.pci_devices[*id].pd_irq =
@@ -219,7 +239,7 @@ pci_init(void)
if (pci_add_device(&id, PCI_VENDOR_OPENBSD, PCI_PRODUCT_OPENBSD_PCHB,
PCI_CLASS_BRIDGE, PCI_SUBCLASS_BRIDGE_HOST,
- PCI_VENDOR_OPENBSD, 0, 0, NULL)) {
+ PCI_VENDOR_OPENBSD, 0, 0, NULL, NULL)) {
log_warnx("%s: can't add PCI host bridge", __progname);
return;
}
@@ -330,7 +350,7 @@ pci_handle_data_reg(struct vm_run_params *vrp)
csfunc = pci.pci_devices[d].pd_csfunc;
if (csfunc != NULL) {
- ret = csfunc(vei->vei.vei_dir, (o / 4), &vei->vei.vei_data);
+ ret = csfunc(vei->vei.vei_dir, (o / 4), sz, &vei->vei.vei_data,
pci.pci_devices[d].pd_cookie);
if (ret)
log_warnx("cfg space access function failed for "
"pci device %d", d);
diff --git a/usr.sbin/vmd/pci.h b/usr.sbin/vmd/pci.h
index 01902d77d..a9d7336ed 100644
--- a/usr.sbin/vmd/pci.h
+++ b/usr.sbin/vmd/pci.h
@@ -27,48 +27,50 @@
#define PCI_MAX_PIC_IRQS 10
-typedef int (*pci_cs_fn_t)(int dir, uint8_t reg, uint32_t *data);
+typedef int (*pci_cs_fn_t)(int dir, uint8_t reg, uint8_t sz, uint32_t *data,
void *cookie);
typedef int (*pci_iobar_fn_t)(int dir, uint16_t reg, uint32_t *data, uint8_t *,
void *, uint8_t);
typedef int (*pci_mmiobar_fn_t)(int dir, uint32_t ofs, uint32_t *data);
-union pci_dev {
- uint32_t pd_cfg_space[PCI_CONFIG_SPACE_SIZE / 4];
- struct {
- uint16_t pd_vid;
- uint16_t pd_did;
- uint16_t pd_cmd;
- uint16_t pd_status;
- uint8_t pd_rev;
- uint8_t pd_prog_if;
- uint8_t pd_subclass;
- uint8_t pd_class;
- uint8_t pd_cache_size;
- uint8_t pd_lat_timer;
- uint8_t pd_header_type;
- uint8_t pd_bist;
- uint32_t pd_bar[PCI_MAX_BARS];
- uint32_t pd_cardbus_cis;
- uint16_t pd_subsys_vid;
- uint16_t pd_subsys_id;
- uint32_t pd_exp_rom_addr;
- uint8_t pd_cap;
- uint32_t pd_reserved0 : 24;
- uint32_t pd_reserved1;
- uint8_t pd_irq;
- uint8_t pd_int;
- uint8_t pd_min_grant;
- uint8_t pd_max_grant;
+struct pci_dev {
+ union {
+ uint32_t pd_cfg_space[PCI_CONFIG_SPACE_SIZE / 4];
+ struct {
+ uint16_t pd_vid;
+ uint16_t pd_did;
+ uint16_t pd_cmd;
+ uint16_t pd_status;
+ uint8_t pd_rev;
+ uint8_t pd_prog_if;
+ uint8_t pd_subclass;
+ uint8_t pd_class;
+ uint8_t pd_cache_size;
+ uint8_t pd_lat_timer;
+ uint8_t pd_header_type;
+ uint8_t pd_bist;
+ uint32_t pd_bar[PCI_MAX_BARS];
+ uint32_t pd_cardbus_cis;
+ uint16_t pd_subsys_vid;
+ uint16_t pd_subsys_id;
+ uint32_t pd_exp_rom_addr;
+ uint8_t pd_cap;
+ uint32_t pd_reserved0 : 24;
+ uint32_t pd_reserved1;
+ uint8_t pd_irq;
+ uint8_t pd_int;
+ uint8_t pd_min_grant;
+ uint8_t pd_max_grant;
+ } __packed;
+ };
+ uint8_t pd_bar_ct;
+ pci_cs_fn_t pd_csfunc;
- uint8_t pd_bar_ct;
- pci_cs_fn_t pd_csfunc;
-
- uint8_t pd_bartype[PCI_MAX_BARS];
- uint32_t pd_barsize[PCI_MAX_BARS];
- void *pd_barfunc[PCI_MAX_BARS];
- void *pd_bar_cookie[PCI_MAX_BARS];
- } __packed;
+ uint8_t pd_bartype[PCI_MAX_BARS];
+ uint32_t pd_barsize[PCI_MAX_BARS];
+ void *pd_barfunc[PCI_MAX_BARS];
+ void *pd_bar_cookie[PCI_MAX_BARS];
+ void *pd_cookie;
};
struct pci {
@@ -79,7 +81,7 @@ struct pci {
uint32_t pci_addr_reg;
uint32_t pci_data_reg;
- union pci_dev pci_devices[PCI_CONFIG_MAX_DEV];
+ struct pci_dev pci_devices[PCI_CONFIG_MAX_DEV];
};
void pci_handle_address_reg(struct vm_run_params *);
@@ -87,8 +89,8 @@ void pci_handle_data_reg(struct vm_run_params *);
uint8_t pci_handle_io(struct vm_run_params *);
void pci_init(void);
int pci_add_device(uint8_t *, uint16_t, uint16_t, uint8_t, uint8_t, uint16_t,
- uint16_t, uint8_t, pci_cs_fn_t);
-int pci_add_bar(uint8_t, uint32_t, void *, void *);
+ uint16_t, uint8_t, pci_cs_fn_t, void *);
+int pci_add_bar(uint8_t, uint32_t, uint32_t, void *, void *);
int pci_set_bar_fn(uint8_t, uint8_t, void *, void *);
uint8_t pci_get_dev_irq(uint8_t);
int pci_dump(int);
diff --git a/usr.sbin/vmd/virtio.c b/usr.sbin/vmd/virtio.c
index 8800594fc..430f41995 100644
--- a/usr.sbin/vmd/virtio.c
+++ b/usr.sbin/vmd/virtio.c
@@ -1797,13 +1797,13 @@ virtio_init(struct vmd_vm *vm, int child_cdrom,
PCI_PRODUCT_QUMRANET_VIO_RNG, PCI_CLASS_SYSTEM,
PCI_SUBCLASS_SYSTEM_MISC,
PCI_VENDOR_OPENBSD,
- PCI_PRODUCT_VIRTIO_ENTROPY, 1, NULL)) {
+ PCI_PRODUCT_VIRTIO_ENTROPY, 1, NULL, NULL)) {
log_warnx("%s: can't add PCI virtio rng device",
__progname);
return;
}
- if (pci_add_bar(id, PCI_MAPREG_TYPE_IO, virtio_rnd_io, NULL)) {
+ if (pci_add_bar(id, PCI_MAPREG_TYPE_IO, VMM_PCI_IO_BAR_SIZE,
virtio_rnd_io, NULL)) {
log_warnx("%s: can't add bar for virtio rng device",
__progname);
return;
@@ -1835,14 +1835,14 @@ virtio_init(struct vmd_vm *vm, int child_cdrom,
PCI_PRODUCT_QUMRANET_VIO_NET, PCI_CLASS_SYSTEM,
PCI_SUBCLASS_SYSTEM_MISC,
PCI_VENDOR_OPENBSD,
- PCI_PRODUCT_VIRTIO_NETWORK, 1, NULL)) {
+ PCI_PRODUCT_VIRTIO_NETWORK, 1, NULL, NULL)) {
log_warnx("%s: can't add PCI virtio net device",
__progname);
return;
}
- if (pci_add_bar(id, PCI_MAPREG_TYPE_IO, virtio_net_io,
- &vionet[i])) {
+ if (pci_add_bar(id, PCI_MAPREG_TYPE_IO,
VMM_PCI_IO_BAR_SIZE,
+ virtio_net_io, &vionet[i])) {
log_warnx("%s: can't add bar for virtio net "
"device", __progname);
return;
@@ -1923,13 +1923,13 @@ virtio_init(struct vmd_vm *vm, int child_cdrom,
PCI_CLASS_MASS_STORAGE,
PCI_SUBCLASS_MASS_STORAGE_SCSI,
PCI_VENDOR_OPENBSD,
- PCI_PRODUCT_VIRTIO_BLOCK, 1, NULL)) {
+ PCI_PRODUCT_VIRTIO_BLOCK, 1, NULL, NULL)) {
log_warnx("%s: can't add PCI virtio block "
"device", __progname);
return;
}
- if (pci_add_bar(id, PCI_MAPREG_TYPE_IO, virtio_blk_io,
- &vioblk[i])) {
+ if (pci_add_bar(id, PCI_MAPREG_TYPE_IO,
VMM_PCI_IO_BAR_SIZE,
+ virtio_blk_io, &vioblk[i])) {
log_warnx("%s: can't add bar for virtio block "
"device", __progname);
return;
@@ -1971,13 +1971,14 @@ virtio_init(struct vmd_vm *vm, int child_cdrom,
PCI_CLASS_MASS_STORAGE,
PCI_SUBCLASS_MASS_STORAGE_SCSI,
PCI_VENDOR_OPENBSD,
- PCI_PRODUCT_VIRTIO_SCSI, 1, NULL)) {
+ PCI_PRODUCT_VIRTIO_SCSI, 1, NULL, NULL)) {
log_warnx("%s: can't add PCI vioscsi device",
__progname);
return;
}
- if (pci_add_bar(id, PCI_MAPREG_TYPE_IO, vioscsi_io, vioscsi)) {
+ if (pci_add_bar(id, PCI_MAPREG_TYPE_IO, VMM_PCI_IO_BAR_SIZE,
+ vioscsi_io, vioscsi)) {
log_warnx("%s: can't add bar for vioscsi device",
__progname);
return;
@@ -2013,13 +2014,13 @@ virtio_init(struct vmd_vm *vm, int child_cdrom,
PCI_CLASS_COMMUNICATIONS,
PCI_SUBCLASS_COMMUNICATIONS_MISC,
PCI_VENDOR_OPENBSD,
- PCI_PRODUCT_VIRTIO_VMMCI, 1, NULL)) {
+ PCI_PRODUCT_VIRTIO_VMMCI, 1, NULL, NULL)) {
log_warnx("%s: can't add PCI vmm control device",
__progname);
return;
}
- if (pci_add_bar(id, PCI_MAPREG_TYPE_IO, vmmci_io, NULL)) {
+ if (pci_add_bar(id, PCI_MAPREG_TYPE_IO, VMM_PCI_IO_BAR_SIZE, vmmci_io,
NULL)) {
log_warnx("%s: can't add bar for vmm control device",
__progname);
return;
--
2.26.2
