[PATCH net-next] virtio: kdoc for struct virtio_pci_modern_device
Finally following up to Simon's suggestion for some kdoc attention
on struct virtio_pci_modern_device.
Link: https://lore.kernel.org/netdev/ze%2fqs0lnuvxfa...@corigine.com/
Cc: Simon Horman
Signed-off-by: Shannon Nelson
Acked-by: Eugenio Pérez
---
include/linux/virtio_pci_modern.h | 34 ---
1 file changed, 22 insertions(+), 12 deletions(-)
diff --git a/include/linux/virtio_pci_modern.h
b/include/linux/virtio_pci_modern.h
index 067ac1d789bc..a38c729d1973 100644
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
@@ -12,37 +12,47 @@ struct virtio_pci_modern_common_cfg {
__le16 queue_reset; /* read-write */
};
+/**
+ * struct virtio_pci_modern_device - info for modern PCI virtio
+ * @pci_dev: Ptr to the PCI device struct
+ * @common:Position of the common capability in the PCI config
+ * @device:Device-specific data (non-legacy mode)
+ * @notify_base:Base of vq notifications (non-legacy mode)
+ * @notify_pa: Physical base of vq notifications
+ * @isr: Where to read and clear interrupt
+ * @notify_len:So we can sanity-check accesses
+ * @device_len:So we can sanity-check accesses
+ * @notify_map_cap: Capability for when we need to map notifications per-vq
+ * @notify_offset_multiplier: Multiply queue_notify_off by this value
+ *(non-legacy mode).
+ * @modern_bars:Bitmask of BARs
+ * @id:Device and vendor id
+ * @device_id_check: Callback defined before vp_modern_probe() to be used to
+ * verify the PCI device is a vendor's expected device rather
+ * than the standard virtio PCI device
+ * Returns the found device id or ERRNO
+ * @dma_mask: Optional mask instead of the traditional DMA_BIT_MASK(64),
+ * for vendor devices with DMA space address limitations
+ */
struct virtio_pci_modern_device {
struct pci_dev *pci_dev;
struct virtio_pci_common_cfg __iomem *common;
- /* Device-specific data (non-legacy mode) */
void __iomem *device;
- /* Base of vq notifications (non-legacy mode). */
void __iomem *notify_base;
- /* Physical base of vq notifications */
resource_size_t notify_pa;
- /* Where to read and clear interrupt */
u8 __iomem *isr;
- /* So we can sanity-check accesses. */
size_t notify_len;
size_t device_len;
- /* Capability for when we need to map notifications per-vq. */
int notify_map_cap;
- /* Multiply queue_notify_off by this value. (non-legacy mode). */
u32 notify_offset_multiplier;
-
int modern_bars;
-
struct virtio_device_id id;
- /* optional check for vendor virtio device, returns dev_id or -ERRNO */
int (*device_id_check)(struct pci_dev *pdev);
-
- /* optional mask for devices with limited DMA space */
u64 dma_mask;
};
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH iproute2] vdpa: consume device_features parameter
From: Allen Hubbe
Consume the parameter to device_features when parsing command line
options. Otherwise the parameter may be used again as an option name.
# vdpa dev add ... device_features 0xdeadbeef mac 00:11:22:33:44:55
Unknown option "0xdeadbeef"
Fixes: a4442ce58ebb ("vdpa: allow provisioning device features")
Signed-off-by: Allen Hubbe
Reviewed-by: Shannon Nelson
Reviewed-by: Si-Wei Liu
---
vdpa/vdpa.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/vdpa/vdpa.c b/vdpa/vdpa.c
index 8bbe452c..6e4a9c11 100644
--- a/vdpa/vdpa.c
+++ b/vdpa/vdpa.c
@@ -353,6 +353,8 @@ static int vdpa_argv_parse(struct vdpa *vdpa, int argc,
char **argv,
>device_features);
if (err)
return err;
+
+ NEXT_ARG_FWD();
o_found |= VDPA_OPT_VDEV_FEATURES;
} else {
fprintf(stderr, "Unknown option \"%s\"\n", *argv);
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH net] virtio: kdoc for struct virtio_pci_modern_device
Finally following up to Simon's suggestion for some kdoc attention
on struct virtio_pci_modern_device.
Link: https://lore.kernel.org/netdev/ze%2fqs0lnuvxfa...@corigine.com/
Cc: Simon Horman
Signed-off-by: Shannon Nelson
---
include/linux/virtio_pci_modern.h | 34 ---
1 file changed, 22 insertions(+), 12 deletions(-)
diff --git a/include/linux/virtio_pci_modern.h
b/include/linux/virtio_pci_modern.h
index 067ac1d789bc..a38c729d1973 100644
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
@@ -12,37 +12,47 @@ struct virtio_pci_modern_common_cfg {
__le16 queue_reset; /* read-write */
};
+/**
+ * struct virtio_pci_modern_device - info for modern PCI virtio
+ * @pci_dev: Ptr to the PCI device struct
+ * @common:Position of the common capability in the PCI config
+ * @device:Device-specific data (non-legacy mode)
+ * @notify_base:Base of vq notifications (non-legacy mode)
+ * @notify_pa: Physical base of vq notifications
+ * @isr: Where to read and clear interrupt
+ * @notify_len:So we can sanity-check accesses
+ * @device_len:So we can sanity-check accesses
+ * @notify_map_cap: Capability for when we need to map notifications per-vq
+ * @notify_offset_multiplier: Multiply queue_notify_off by this value
+ *(non-legacy mode).
+ * @modern_bars:Bitmask of BARs
+ * @id:Device and vendor id
+ * @device_id_check: Callback defined before vp_modern_probe() to be used to
+ * verify the PCI device is a vendor's expected device rather
+ * than the standard virtio PCI device
+ * Returns the found device id or ERRNO
+ * @dma_mask: Optional mask instead of the traditional DMA_BIT_MASK(64),
+ * for vendor devices with DMA space address limitations
+ */
struct virtio_pci_modern_device {
struct pci_dev *pci_dev;
struct virtio_pci_common_cfg __iomem *common;
- /* Device-specific data (non-legacy mode) */
void __iomem *device;
- /* Base of vq notifications (non-legacy mode). */
void __iomem *notify_base;
- /* Physical base of vq notifications */
resource_size_t notify_pa;
- /* Where to read and clear interrupt */
u8 __iomem *isr;
- /* So we can sanity-check accesses. */
size_t notify_len;
size_t device_len;
- /* Capability for when we need to map notifications per-vq. */
int notify_map_cap;
- /* Multiply queue_notify_off by this value. (non-legacy mode). */
u32 notify_offset_multiplier;
-
int modern_bars;
-
struct virtio_device_id id;
- /* optional check for vendor virtio device, returns dev_id or -ERRNO */
int (*device_id_check)(struct pci_dev *pdev);
-
- /* optional mask for devices with limited DMA space */
u64 dma_mask;
};
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH net-next v4 2/2] virtio-net: add cond_resched() to the command waiting loop
On 7/20/23 1:38 AM, Jason Wang wrote:
Adding cond_resched() to the command waiting loop for a better
co-operation with the scheduler. This allows to give CPU a breath to
run other task(workqueue) instead of busy looping when preemption is
not allowed on a device whose CVQ might be slow.
Signed-off-by: Jason Wang
This still leaves hung processes, but at least it doesn't pin the CPU
any more. Thanks.
Reviewed-by: Shannon Nelson
---
drivers/net/virtio_net.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 9f3b1d6ac33d..e7533f29b219 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -2314,8 +2314,10 @@ static bool virtnet_send_command(struct virtnet_info
*vi, u8 class, u8 cmd,
* into the hypervisor, so the request should be handled immediately.
*/
while (!virtqueue_get_buf(vi->cvq, ) &&
- !virtqueue_is_broken(vi->cvq))
+ !virtqueue_is_broken(vi->cvq)) {
+ cond_resched();
cpu_relax();
+ }
return vi->ctrl->status == VIRTIO_NET_OK;
}
--
2.39.3
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH net-next v4 1/2] virtio-net: convert rx mode setting to use workqueue
On 7/20/23 1:38 AM, Jason Wang wrote:
This patch convert rx mode setting to be done in a workqueue, this is
a must for allow to sleep when waiting for the cvq command to
response since current code is executed under addr spin lock.
Note that we need to disable and flush the workqueue during freeze,
this means the rx mode setting is lost after resuming. This is not the
bug of this patch as we never try to restore rx mode setting during
resume.
Signed-off-by: Jason Wang
Reviewed-by: Shannon Nelson
---
drivers/net/virtio_net.c | 55 +---
1 file changed, 52 insertions(+), 3 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index d67b36fdba0d..9f3b1d6ac33d 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -274,6 +274,12 @@ struct virtnet_info {
/* Work struct for config space updates */
struct work_struct config_work;
+ /* Work struct for setting rx mode */
+ struct work_struct rx_mode_work;
+
+ /* OK to queue work setting RX mode? */
+ bool rx_mode_work_enabled;
+
/* Does the affinity hint is set for virtqueues? */
bool affinity_hint_set;
@@ -397,6 +403,20 @@ static void disable_delayed_refill(struct virtnet_info *vi)
spin_unlock_bh(>refill_lock);
}
+static void enable_rx_mode_work(struct virtnet_info *vi)
+{
+ rtnl_lock();
+ vi->rx_mode_work_enabled = true;
+ rtnl_unlock();
+}
+
+static void disable_rx_mode_work(struct virtnet_info *vi)
+{
+ rtnl_lock();
+ vi->rx_mode_work_enabled = false;
+ rtnl_unlock();
+}
+
static void virtqueue_napi_schedule(struct napi_struct *napi,
struct virtqueue *vq)
{
@@ -2448,9 +2468,11 @@ static int virtnet_close(struct net_device *dev)
return 0;
}
-static void virtnet_set_rx_mode(struct net_device *dev)
+static void virtnet_rx_mode_work(struct work_struct *work)
{
- struct virtnet_info *vi = netdev_priv(dev);
+ struct virtnet_info *vi =
+ container_of(work, struct virtnet_info, rx_mode_work);
+ struct net_device *dev = vi->dev;
struct scatterlist sg[2];
struct virtio_net_ctrl_mac *mac_data;
struct netdev_hw_addr *ha;
@@ -2463,6 +2485,8 @@ static void virtnet_set_rx_mode(struct net_device *dev)
if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
return;
+ rtnl_lock();
+
vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0);
vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
@@ -2480,14 +2504,19 @@ static void virtnet_set_rx_mode(struct net_device *dev)
dev_warn(>dev, "Failed to %sable allmulti mode.\n",
vi->ctrl->allmulti ? "en" : "dis");
+ netif_addr_lock_bh(dev);
+
uc_count = netdev_uc_count(dev);
mc_count = netdev_mc_count(dev);
/* MAC filter - use one buffer for both lists */
buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
(2 * sizeof(mac_data->entries)), GFP_ATOMIC);
mac_data = buf;
- if (!buf)
+ if (!buf) {
+ netif_addr_unlock_bh(dev);
+ rtnl_unlock();
return;
+ }
sg_init_table(sg, 2);
@@ -2508,6 +2537,8 @@ static void virtnet_set_rx_mode(struct net_device *dev)
netdev_for_each_mc_addr(ha, dev)
memcpy(_data->macs[i++][0], ha->addr, ETH_ALEN);
+ netif_addr_unlock_bh(dev);
+
sg_set_buf([1], mac_data,
sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
@@ -2515,9 +2546,19 @@ static void virtnet_set_rx_mode(struct net_device *dev)
VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
dev_warn(>dev, "Failed to set MAC filter table.\n");
+ rtnl_unlock();
+
kfree(buf);
}
+static void virtnet_set_rx_mode(struct net_device *dev)
+{
+ struct virtnet_info *vi = netdev_priv(dev);
+
+ if (vi->rx_mode_work_enabled)
+ schedule_work(>rx_mode_work);
+}
+
static int virtnet_vlan_rx_add_vid(struct net_device *dev,
__be16 proto, u16 vid)
{
@@ -3286,6 +3327,8 @@ static void virtnet_freeze_down(struct virtio_device
*vdev)
/* Make sure no work handler is accessing the device */
flush_work(>config_work);
+ disable_rx_mode_work(vi);
+ flush_work(>rx_mode_work);
netif_tx_lock_bh(vi->dev);
netif_device_detach(vi->dev);
@@ -3308,6 +3351,7 @@ static int virtnet_restore_up(struct virtio_device *vdev)
virtio_device_ready(vdev);
enable_delayed_refill(vi);
+ enable_rx_mode_work(vi);
if (netif_running(vi->dev)) {
err = virtnet_open(vi->dev);
@@ -4121,6 +4165,7 @@ static int virtnet_probe(struct virtio_device *vdev)
vdev->priv = vi;
Re: [PATCH net-next v4 2/2] virtio-net: add cond_resched() to the command waiting loop
On 7/20/23 1:38 AM, Jason Wang wrote:
Adding cond_resched() to the command waiting loop for a better
co-operation with the scheduler. This allows to give CPU a breath to
run other task(workqueue) instead of busy looping when preemption is
not allowed on a device whose CVQ might be slow.
Signed-off-by: Jason Wang
---
drivers/net/virtio_net.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 9f3b1d6ac33d..e7533f29b219 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -2314,8 +2314,10 @@ static bool virtnet_send_command(struct virtnet_info
*vi, u8 class, u8 cmd,
* into the hypervisor, so the request should be handled immediately.
*/
while (!virtqueue_get_buf(vi->cvq, ) &&
- !virtqueue_is_broken(vi->cvq))
+ !virtqueue_is_broken(vi->cvq)) {
+ cond_resched();
cpu_relax();
+ }
The cover letter suggests that this addresses the infinite poll for
buggy devices, but I don't see how that is resolved here. This should
make it a little nicer to the system, but it still is going to poll
forever on a device that has gone catatonic. Is there a reason that I'm
missing that we don't have a polling limit here?
sln
return vi->ctrl->status == VIRTIO_NET_OK;
}
--
2.39.3
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH] vpda: pds: fix building without debugfs
On 7/19/23 12:40 PM, Arnd Bergmann wrote:
From: Arnd Bergmann
The new debugfs code still gets called when debugfs is disabled, resulting
in a link failure:
ERROR: modpost: "pds_vdpa_debugfs_del_vdpadev" [drivers/vdpa/pds/pds_vdpa.ko]
undefined!
ERROR: modpost: "pds_vdpa_debugfs_add_vdpadev" [drivers/vdpa/pds/pds_vdpa.ko]
undefined!
ERROR: modpost: "pds_vdpa_debugfs_reset_vdpadev" [drivers/vdpa/pds/pds_vdpa.ko]
undefined!
ERROR: modpost: "pds_vdpa_debugfs_create" [drivers/vdpa/pds/pds_vdpa.ko]
undefined!
ERROR: modpost: "pds_vdpa_debugfs_add_ident" [drivers/vdpa/pds/pds_vdpa.ko]
undefined!
ERROR: modpost: "pds_vdpa_debugfs_destroy" [drivers/vdpa/pds/pds_vdpa.ko]
undefined!
ERROR: modpost: "pds_vdpa_debugfs_add_pcidev" [drivers/vdpa/pds/pds_vdpa.ko]
undefined!
Add the usual #if/#else block around the declarations to supply empty stubs
for the broken case.
Fixes: 151cc834f3dda ("pds_vdpa: add support for vdpa and vdpamgmt interfaces")
Signed-off-by: Arnd Bergmann
Thanks, Arnd. There is a patch already intended to address this:
https://lore.kernel.org/netdev/20230706231718.54198-1-shannon.nel...@amd.com/
Maybe it needs to get pulled in through a different tree to show up quicker?
sln
---
drivers/vdpa/pds/debugfs.h | 10 ++
1 file changed, 10 insertions(+)
diff --git a/drivers/vdpa/pds/debugfs.h b/drivers/vdpa/pds/debugfs.h
index c088a4e8f1e92..6fbd64ace13a3 100644
--- a/drivers/vdpa/pds/debugfs.h
+++ b/drivers/vdpa/pds/debugfs.h
@@ -6,6 +6,7 @@
#include
+#ifdef CONFIG_DEBUG_FS
void pds_vdpa_debugfs_create(void);
void pds_vdpa_debugfs_destroy(void);
void pds_vdpa_debugfs_add_pcidev(struct pds_vdpa_aux *vdpa_aux);
@@ -13,5 +14,14 @@ void pds_vdpa_debugfs_add_ident(struct pds_vdpa_aux
*vdpa_aux);
void pds_vdpa_debugfs_add_vdpadev(struct pds_vdpa_aux *vdpa_aux);
void pds_vdpa_debugfs_del_vdpadev(struct pds_vdpa_aux *vdpa_aux);
void pds_vdpa_debugfs_reset_vdpadev(struct pds_vdpa_aux *vdpa_aux);
+#else
+static inline void pds_vdpa_debugfs_create(void) { }
+static inline void pds_vdpa_debugfs_destroy(void) { }
+static inline void pds_vdpa_debugfs_add_pcidev(struct pds_vdpa_aux *vdpa_aux)
{ }
+static inline void pds_vdpa_debugfs_add_ident(struct pds_vdpa_aux *vdpa_aux) {
}
+static inline void pds_vdpa_debugfs_add_vdpadev(struct pds_vdpa_aux *vdpa_aux)
{ }
+static inline void pds_vdpa_debugfs_del_vdpadev(struct pds_vdpa_aux *vdpa_aux)
{ }
+static inline void pds_vdpa_debugfs_reset_vdpadev(struct pds_vdpa_aux
*vdpa_aux) { }
+#endif
#endif /* _PDS_VDPA_DEBUGFS_H_ */
--
2.39.2
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH RFC net] virtio-net: add timeout for virtnet_send_command()
When trying to talk to a device that has gone out to lunch, the
virtnet_send_command() will sit and spin forever, causing a soft
lockup and eventually crashing the kernel. Add a limit to the
spin and return false if we hit the timeout. The 2 second time
limit seems a bit arbitrary, but a reasonable place to start.
This is a little more brute force than Jason's suggestions in [1],
but at least prevents the soft lockups and eventual kernel crash
that we were seeing in testing.
[1]: https://lore.kernel.org/netdev/20230524081842.3060-1-jasow...@redhat.com/
Fixes: 2a41f71d3bd9 ("virtio_net: Add a virtqueue for outbound control
commands")
Signed-off-by: Shannon Nelson
---
drivers/net/virtio_net.c | 11 +--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 0db14f6b87d3..c3bf1c9f3244 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -2264,6 +2264,8 @@ static bool virtnet_send_command(struct virtnet_info *vi,
u8 class, u8 cmd,
{
struct scatterlist *sgs[4], hdr, stat;
unsigned out_num = 0, tmp;
+ unsigned long deadline;
+ bool timeout;
int ret;
/* Caller should know better */
@@ -2297,11 +2299,16 @@ static bool virtnet_send_command(struct virtnet_info
*vi, u8 class, u8 cmd,
/* Spin for a response, the kick causes an ioport write, trapping
* into the hypervisor, so the request should be handled immediately.
*/
+ deadline = jiffies + 2 * HZ;
+ timeout = false;
while (!virtqueue_get_buf(vi->cvq, ) &&
- !virtqueue_is_broken(vi->cvq))
+ !virtqueue_is_broken(vi->cvq) &&
+ !timeout) {
cpu_relax();
+ timeout = time_after(jiffies, deadline);
+ }
- return vi->ctrl->status == VIRTIO_NET_OK;
+ return vi->ctrl->status == VIRTIO_NET_OK && !timeout;
}
static int virtnet_set_mac_address(struct net_device *dev, void *p)
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v2 virtio 3/5] pds_vdpa: clean and reset vqs entries
Make sure that we initialize the vqs[] entries the same
way both for initial setup and after a vq reset.
Fixes: 151cc834f3dd ("pds_vdpa: add support for vdpa and vdpamgmt interfaces")
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/vdpa_dev.c | 24 +---
1 file changed, 17 insertions(+), 7 deletions(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index 5b566e0eef0a..04a362648b02 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -428,6 +428,17 @@ static void pds_vdpa_set_status(struct vdpa_device
*vdpa_dev, u8 status)
}
}
+static void pds_vdpa_init_vqs_entry(struct pds_vdpa_device *pdsv, int qid,
+ void __iomem *notify)
+{
+ memset(>vqs[qid], 0, sizeof(pdsv->vqs[0]));
+ pdsv->vqs[qid].qid = qid;
+ pdsv->vqs[qid].pdsv = pdsv;
+ pdsv->vqs[qid].ready = false;
+ pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR;
+ pdsv->vqs[qid].notify = notify;
+}
+
static int pds_vdpa_reset(struct vdpa_device *vdpa_dev)
{
struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
@@ -450,8 +461,7 @@ static int pds_vdpa_reset(struct vdpa_device *vdpa_dev)
dev_err(dev, "%s: reset_vq failed qid %d:
%pe\n",
__func__, i, ERR_PTR(err));
pds_vdpa_release_irq(pdsv, i);
- memset(>vqs[i], 0, sizeof(pdsv->vqs[0]));
- pdsv->vqs[i].ready = false;
+ pds_vdpa_init_vqs_entry(pdsv, i, pdsv->vqs[i].notify);
}
}
@@ -640,11 +650,11 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
for (i = 0; i < pdsv->num_vqs; i++) {
- pdsv->vqs[i].qid = i;
- pdsv->vqs[i].pdsv = pdsv;
- pdsv->vqs[i].irq = VIRTIO_MSI_NO_VECTOR;
- pdsv->vqs[i].notify =
vp_modern_map_vq_notify(>vdpa_aux->vd_mdev,
- i,
>vqs[i].notify_pa);
+ void __iomem *notify;
+
+ notify = vp_modern_map_vq_notify(>vdpa_aux->vd_mdev,
+i, >vqs[i].notify_pa);
+ pds_vdpa_init_vqs_entry(pdsv, i, notify);
}
pdsv->vdpa_dev.mdev = _aux->vdpa_mdev;
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v2 virtio 2/5] pds_vdpa: always allow offering VIRTIO_NET_F_MAC
Our driver sets a mac if the HW is 00:..:00 so we need to be sure to
advertise VIRTIO_NET_F_MAC even if the HW doesn't. We also need to be
sure that virtio_net sees the VIRTIO_NET_F_MAC and doesn't rewrite the
mac address that a user may have set with the vdpa utility.
After reading the hw_feature bits, add the VIRTIO_NET_F_MAC to the driver's
supported_features and use that for reporting what is available. If the
HW is not advertising it, be sure to strip the VIRTIO_NET_F_MAC before
finishing the feature negotiation. If the user specifies a device_features
bitpattern in the vdpa utility without the VIRTIO_NET_F_MAC set, then
don't set the mac.
Fixes: 151cc834f3dd ("pds_vdpa: add support for vdpa and vdpamgmt interfaces")
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/debugfs.c | 2 --
drivers/vdpa/pds/vdpa_dev.c | 30 +-
drivers/vdpa/pds/vdpa_dev.h | 4 ++--
3 files changed, 23 insertions(+), 13 deletions(-)
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
index 21a0dc0cb607..754ccb7a 100644
--- a/drivers/vdpa/pds/debugfs.c
+++ b/drivers/vdpa/pds/debugfs.c
@@ -224,8 +224,6 @@ static int config_show(struct seq_file *seq, void *v)
seq_printf(seq, "dev_status: %#x\n", status);
print_status_bits(seq, status);
- seq_printf(seq, "req_features: %#llx\n", pdsv->req_features);
- print_feature_bits_all(seq, pdsv->req_features);
driver_features =
vp_modern_get_driver_features(>vdpa_aux->vd_mdev);
seq_printf(seq, "driver_features: %#llx\n", driver_features);
print_feature_bits_all(seq, driver_features);
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index e2e99bb0be2b..5b566e0eef0a 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -318,6 +318,7 @@ static int pds_vdpa_set_driver_features(struct vdpa_device
*vdpa_dev, u64 featur
struct device *dev = >vdpa_dev.dev;
u64 driver_features;
u64 nego_features;
+ u64 hw_features;
u64 missing;
if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)) && features) {
@@ -325,21 +326,26 @@ static int pds_vdpa_set_driver_features(struct
vdpa_device *vdpa_dev, u64 featur
return -EOPNOTSUPP;
}
- pdsv->req_features = features;
-
/* Check for valid feature bits */
- nego_features = features &
le64_to_cpu(pdsv->vdpa_aux->ident.hw_features);
- missing = pdsv->req_features & ~nego_features;
+ nego_features = features & pdsv->supported_features;
+ missing = features & ~nego_features;
if (missing) {
dev_err(dev, "Can't support all requested features in %#llx,
missing %#llx features\n",
- pdsv->req_features, missing);
+ features, missing);
return -EOPNOTSUPP;
}
+ pdsv->negotiated_features = nego_features;
+
driver_features = pds_vdpa_get_driver_features(vdpa_dev);
dev_dbg(dev, "%s: %#llx => %#llx\n",
__func__, driver_features, nego_features);
+ /* if we're faking the F_MAC, strip it before writing to device */
+ hw_features = le64_to_cpu(pdsv->vdpa_aux->ident.hw_features);
+ if (!(hw_features & BIT_ULL(VIRTIO_NET_F_MAC)))
+ nego_features &= ~BIT_ULL(VIRTIO_NET_F_MAC);
+
if (driver_features == nego_features)
return 0;
@@ -352,7 +358,7 @@ static u64 pds_vdpa_get_driver_features(struct vdpa_device
*vdpa_dev)
{
struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
- return vp_modern_get_driver_features(>vdpa_aux->vd_mdev);
+ return pdsv->negotiated_features;
}
static void pds_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,
@@ -564,7 +570,7 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
u64 unsupp_features =
- add_config->device_features & ~mgmt->supported_features;
+ add_config->device_features & ~pdsv->supported_features;
if (unsupp_features) {
dev_err(dev, "Unsupported features: %#llx\n",
unsupp_features);
@@ -615,7 +621,8 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
}
/* Set a mac, either from the user config if provided
-* or set a random mac if default is 00:..:00
+* or use the device's mac if not 00:..:00
+* or set a random mac
*/
if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
ether_addr_copy(pdsv->mac, add_config->net.mac);
@@ -624,7 +631,8 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
vc = pdsv->vdpa_aux->vd_mdev.device;
memcpy_fromio(pdsv->mac, vc->mac, sizeof(pdsv->mac));
-
[PATCH v2 virtio 4/5] pds_vdpa: alloc irq vectors on DRIVER_OK
From: Allen Hubbe
We were allocating irq vectors at the time the aux dev was probed,
but that is before the PCI VF is assigned to a separate iommu domain
by vhost_vdpa. Because vhost_vdpa later changes the iommu domain the
interrupts do not work.
Instead, we can allocate the irq vectors later when we see DRIVER_OK and
know that the reassignment of the PCI VF to an iommu domain has already
happened.
Fixes: 151cc834f3dd ("pds_vdpa: add support for vdpa and vdpamgmt interfaces")
Signed-off-by: Allen Hubbe
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/pds/vdpa_dev.c | 110 ++--
1 file changed, 81 insertions(+), 29 deletions(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index 04a362648b02..52b2449182ad 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -126,11 +126,9 @@ static void pds_vdpa_release_irq(struct pds_vdpa_device
*pdsv, int qid)
static void pds_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool
ready)
{
struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
- struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;
struct device *dev = >vdpa_dev.dev;
u64 driver_features;
u16 invert_idx = 0;
- int irq;
int err;
dev_dbg(dev, "%s: qid %d ready %d => %d\n",
@@ -143,19 +141,6 @@ static void pds_vdpa_set_vq_ready(struct vdpa_device
*vdpa_dev, u16 qid, bool re
invert_idx = PDS_VDPA_PACKED_INVERT_IDX;
if (ready) {
- irq = pci_irq_vector(pdev, qid);
- snprintf(pdsv->vqs[qid].irq_name,
sizeof(pdsv->vqs[qid].irq_name),
-"vdpa-%s-%d", dev_name(dev), qid);
-
- err = request_irq(irq, pds_vdpa_isr, 0,
- pdsv->vqs[qid].irq_name, >vqs[qid]);
- if (err) {
- dev_err(dev, "%s: no irq for qid %d: %pe\n",
- __func__, qid, ERR_PTR(err));
- return;
- }
- pdsv->vqs[qid].irq = irq;
-
/* Pass vq setup info to DSC using adminq to gather up and
* send all info at once so FW can do its full set up in
* one easy operation
@@ -164,7 +149,6 @@ static void pds_vdpa_set_vq_ready(struct vdpa_device
*vdpa_dev, u16 qid, bool re
if (err) {
dev_err(dev, "Failed to init vq %d: %pe\n",
qid, ERR_PTR(err));
- pds_vdpa_release_irq(pdsv, qid);
ready = false;
}
} else {
@@ -172,7 +156,6 @@ static void pds_vdpa_set_vq_ready(struct vdpa_device
*vdpa_dev, u16 qid, bool re
if (err)
dev_err(dev, "%s: reset_vq failed qid %d: %pe\n",
__func__, qid, ERR_PTR(err));
- pds_vdpa_release_irq(pdsv, qid);
}
pdsv->vqs[qid].ready = ready;
@@ -395,6 +378,72 @@ static u8 pds_vdpa_get_status(struct vdpa_device *vdpa_dev)
return vp_modern_get_status(>vdpa_aux->vd_mdev);
}
+static int pds_vdpa_request_irqs(struct pds_vdpa_device *pdsv)
+{
+ struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;
+ struct pds_vdpa_aux *vdpa_aux = pdsv->vdpa_aux;
+ struct device *dev = >vdpa_dev.dev;
+ int max_vq, nintrs, qid, err;
+
+ max_vq = vdpa_aux->vdpa_mdev.max_supported_vqs;
+
+ nintrs = pci_alloc_irq_vectors(pdev, max_vq, max_vq, PCI_IRQ_MSIX);
+ if (nintrs < 0) {
+ dev_err(dev, "Couldn't get %d msix vectors: %pe\n",
+ max_vq, ERR_PTR(nintrs));
+ return nintrs;
+ }
+
+ for (qid = 0; qid < pdsv->num_vqs; ++qid) {
+ int irq = pci_irq_vector(pdev, qid);
+
+ snprintf(pdsv->vqs[qid].irq_name,
sizeof(pdsv->vqs[qid].irq_name),
+"vdpa-%s-%d", dev_name(dev), qid);
+
+ err = request_irq(irq, pds_vdpa_isr, 0,
+ pdsv->vqs[qid].irq_name,
+ >vqs[qid]);
+ if (err) {
+ dev_err(dev, "%s: no irq for qid %d: %pe\n",
+ __func__, qid, ERR_PTR(err));
+ goto err_release;
+ }
+
+ pdsv->vqs[qid].irq = irq;
+ }
+
+ vdpa_aux->nintrs = nintrs;
+
+ return 0;
+
+err_release:
+ while (qid--)
+ pds_vdpa_release_irq(pdsv, qid);
+
+ pci_free_irq_vectors(pdev);
+
+ vdpa_aux->nintrs = 0;
+
+ return err;
+}
+
+static void pds_vdpa_release_irqs(struct pds_vdpa_device *pdsv)
+{
+ struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;
+ struct pds_vdpa_aux *vdpa_aux = pdsv->vdpa_aux;
+ int qid;
+
+ if (!vdpa_aux->nintrs)
+ return;
+
+ for (qid
[PATCH v2 virtio 0/5] pds_vdpa: mac, reset, and irq updates
These are some fixes for device providing a MAC address, for allocating irq resources later to support vhost use, and for properly cleaning vq info on reset. v2: - check driver feature request against supported_features, and strip the F_MAC bit if needed just before writing to the HW. - report the driver negotiated features in get_driver_features() - only map the notify on initial setup, not on every reset. - added patch 5 to clear up some driver feature debugfs output Allen Hubbe (2): pds_vdpa: reset to vdpa specified mac pds_vdpa: alloc irq vectors on DRIVER_OK Shannon Nelson (3): pds_vdpa: always allow offering VIRTIO_NET_F_MAC pds_vdpa: clean and reset vqs entries pds_vdpa: fix up debugfs feature bit printing drivers/vdpa/pds/debugfs.c | 15 ++- drivers/vdpa/pds/vdpa_dev.c | 176 +--- drivers/vdpa/pds/vdpa_dev.h | 5 +- 3 files changed, 134 insertions(+), 62 deletions(-) -- 2.17.1 ___ Virtualization mailing list Virtualization@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v2 virtio 1/5] pds_vdpa: reset to vdpa specified mac
From: Allen Hubbe
When the vdpa device is reset, also reinitialize it with the mac address
that was assigned when the device was added.
Fixes: 151cc834f3dd ("pds_vdpa: add support for vdpa and vdpamgmt interfaces")
Signed-off-by: Allen Hubbe
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/vdpa_dev.c | 16
drivers/vdpa/pds/vdpa_dev.h | 1 +
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index 5071a4d58f8d..e2e99bb0be2b 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -409,6 +409,8 @@ static void pds_vdpa_set_status(struct vdpa_device
*vdpa_dev, u8 status)
pdsv->vqs[i].avail_idx = 0;
pdsv->vqs[i].used_idx = 0;
}
+
+ pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
}
if (status & ~old_status & VIRTIO_CONFIG_S_FEATURES_OK) {
@@ -532,7 +534,6 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
struct device *dma_dev;
struct pci_dev *pdev;
struct device *dev;
- u8 mac[ETH_ALEN];
int err;
int i;
@@ -617,19 +618,18 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
* or set a random mac if default is 00:..:00
*/
if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
- ether_addr_copy(mac, add_config->net.mac);
- pds_vdpa_cmd_set_mac(pdsv, mac);
+ ether_addr_copy(pdsv->mac, add_config->net.mac);
} else {
struct virtio_net_config __iomem *vc;
vc = pdsv->vdpa_aux->vd_mdev.device;
- memcpy_fromio(mac, vc->mac, sizeof(mac));
- if (is_zero_ether_addr(mac)) {
- eth_random_addr(mac);
- dev_info(dev, "setting random mac %pM\n", mac);
- pds_vdpa_cmd_set_mac(pdsv, mac);
+ memcpy_fromio(pdsv->mac, vc->mac, sizeof(pdsv->mac));
+ if (is_zero_ether_addr(pdsv->mac)) {
+ eth_random_addr(pdsv->mac);
+ dev_info(dev, "setting random mac %pM\n", pdsv->mac);
}
}
+ pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
for (i = 0; i < pdsv->num_vqs; i++) {
pdsv->vqs[i].qid = i;
diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h
index a1bc37de9537..cf02df287fc4 100644
--- a/drivers/vdpa/pds/vdpa_dev.h
+++ b/drivers/vdpa/pds/vdpa_dev.h
@@ -39,6 +39,7 @@ struct pds_vdpa_device {
u64 req_features; /* features requested by vdpa */
u8 vdpa_index; /* rsvd for future subdevice use */
u8 num_vqs; /* num vqs in use */
+ u8 mac[ETH_ALEN]; /* mac selected when the device was
added */
struct vdpa_callback config_cb;
struct notifier_block nb;
};
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v2 virtio 5/5] pds_vdpa: fix up debugfs feature bit printing
Make clearer in debugfs output the difference between the hw
feature bits, the features supported through the driver, and
the features that have been negotiated.
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/debugfs.c | 13 ++---
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
index 754ccb7a..9b04aad6ec35 100644
--- a/drivers/vdpa/pds/debugfs.c
+++ b/drivers/vdpa/pds/debugfs.c
@@ -176,6 +176,7 @@ static int identity_show(struct seq_file *seq, void *v)
{
struct pds_vdpa_aux *vdpa_aux = seq->private;
struct vdpa_mgmt_dev *mgmt;
+ u64 hw_features;
seq_printf(seq, "aux_dev:%s\n",
dev_name(_aux->padev->aux_dev.dev));
@@ -183,8 +184,9 @@ static int identity_show(struct seq_file *seq, void *v)
mgmt = _aux->vdpa_mdev;
seq_printf(seq, "max_vqs:%d\n", mgmt->max_supported_vqs);
seq_printf(seq, "config_attr_mask: %#llx\n", mgmt->config_attr_mask);
- seq_printf(seq, "supported_features: %#llx\n",
mgmt->supported_features);
- print_feature_bits_all(seq, mgmt->supported_features);
+ hw_features = le64_to_cpu(vdpa_aux->ident.hw_features);
+ seq_printf(seq, "hw_features:%#llx\n", hw_features);
+ print_feature_bits_all(seq, hw_features);
return 0;
}
@@ -200,7 +202,6 @@ static int config_show(struct seq_file *seq, void *v)
{
struct pds_vdpa_device *pdsv = seq->private;
struct virtio_net_config vc;
- u64 driver_features;
u8 status;
memcpy_fromio(, pdsv->vdpa_aux->vd_mdev.device,
@@ -223,10 +224,8 @@ static int config_show(struct seq_file *seq, void *v)
status = vp_modern_get_status(>vdpa_aux->vd_mdev);
seq_printf(seq, "dev_status: %#x\n", status);
print_status_bits(seq, status);
-
- driver_features =
vp_modern_get_driver_features(>vdpa_aux->vd_mdev);
- seq_printf(seq, "driver_features: %#llx\n", driver_features);
- print_feature_bits_all(seq, driver_features);
+ seq_printf(seq, "negotiated_features: %#llx\n",
pdsv->negotiated_features);
+ print_feature_bits_all(seq, pdsv->negotiated_features);
seq_printf(seq, "vdpa_index: %d\n", pdsv->vdpa_index);
seq_printf(seq, "num_vqs: %d\n", pdsv->num_vqs);
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH virtio 1/4] pds_vdpa: reset to vdpa specified mac
On 7/9/23 8:04 PM, Jason Wang wrote:
On Sat, Jul 8, 2023 at 4:12 AM Shannon Nelson wrote:
On 7/7/23 12:33 AM, Jason Wang wrote:
Caution: This message originated from an External Source. Use proper caution
when opening attachments, clicking links, or responding.
On Fri, Jun 30, 2023 at 8:36 AM Shannon Nelson wrote:
From: Allen Hubbe
When the vdpa device is reset, also reinitialize it with the mac address
that was assigned when the device was added.
Fixes: 151cc834f3dd ("pds_vdpa: add support for vdpa and vdpamgmt interfaces")
Signed-off-by: Allen Hubbe
Signed-off-by: Shannon Nelson
Reviewed-by: Brett Creeley
---
drivers/vdpa/pds/vdpa_dev.c | 16
drivers/vdpa/pds/vdpa_dev.h | 1 +
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index 5071a4d58f8d..e2e99bb0be2b 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -409,6 +409,8 @@ static void pds_vdpa_set_status(struct vdpa_device
*vdpa_dev, u8 status)
pdsv->vqs[i].avail_idx = 0;
pdsv->vqs[i].used_idx = 0;
}
+
+ pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
So this is not necessarily called during reset. So I think we need to
move it to pds_vdpa_reset()?
pds_vdpa_reset() calls pds_vdpa_set_status() with a status=0, so this is
already covered.
Yes, but pds_vdpa_set_status() will be called when status is not zero?
Yes, but the set_mac is only done with status==0, whether called by
reset or called when some other thing calls set_status with status==0.
sln
Thanks
sln
The rest looks good.
Thanks
}
if (status & ~old_status & VIRTIO_CONFIG_S_FEATURES_OK) {
@@ -532,7 +534,6 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
struct device *dma_dev;
struct pci_dev *pdev;
struct device *dev;
- u8 mac[ETH_ALEN];
int err;
int i;
@@ -617,19 +618,18 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
* or set a random mac if default is 00:..:00
*/
if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
- ether_addr_copy(mac, add_config->net.mac);
- pds_vdpa_cmd_set_mac(pdsv, mac);
+ ether_addr_copy(pdsv->mac, add_config->net.mac);
} else {
struct virtio_net_config __iomem *vc;
vc = pdsv->vdpa_aux->vd_mdev.device;
- memcpy_fromio(mac, vc->mac, sizeof(mac));
- if (is_zero_ether_addr(mac)) {
- eth_random_addr(mac);
- dev_info(dev, "setting random mac %pM\n", mac);
- pds_vdpa_cmd_set_mac(pdsv, mac);
+ memcpy_fromio(pdsv->mac, vc->mac, sizeof(pdsv->mac));
+ if (is_zero_ether_addr(pdsv->mac)) {
+ eth_random_addr(pdsv->mac);
+ dev_info(dev, "setting random mac %pM\n", pdsv->mac);
}
}
+ pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
for (i = 0; i < pdsv->num_vqs; i++) {
pdsv->vqs[i].qid = i;
diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h
index a1bc37de9537..cf02df287fc4 100644
--- a/drivers/vdpa/pds/vdpa_dev.h
+++ b/drivers/vdpa/pds/vdpa_dev.h
@@ -39,6 +39,7 @@ struct pds_vdpa_device {
u64 req_features; /* features requested by vdpa */
u8 vdpa_index; /* rsvd for future subdevice use */
u8 num_vqs; /* num vqs in use */
+ u8 mac[ETH_ALEN]; /* mac selected when the device was
added */
struct vdpa_callback config_cb;
struct notifier_block nb;
};
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH virtio 3/4] pds_vdpa: clean and reset vqs entries
On 7/7/23 12:36 AM, Jason Wang wrote:
Caution: This message originated from an External Source. Use proper caution
when opening attachments, clicking links, or responding.
On Fri, Jun 30, 2023 at 8:36 AM Shannon Nelson wrote:
Make sure that we initialize the vqs[] entries the same
way both for initial setup and after a vq reset.
Fixes: 151cc834f3dd ("pds_vdpa: add support for vdpa and vdpamgmt interfaces")
Signed-off-by: Shannon Nelson
Reviewed-by: Brett Creeley
---
drivers/vdpa/pds/vdpa_dev.c | 24 +++-
1 file changed, 15 insertions(+), 9 deletions(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index 5e761d625ef3..5e1046c9af3d 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -429,6 +429,18 @@ static void pds_vdpa_set_status(struct vdpa_device
*vdpa_dev, u8 status)
}
}
+static void pds_vdpa_init_vqs_entry(struct pds_vdpa_device *pdsv, int qid)
+{
+ memset(>vqs[qid], 0, sizeof(pdsv->vqs[0]));
+ pdsv->vqs[qid].qid = qid;
+ pdsv->vqs[qid].pdsv = pdsv;
+ pdsv->vqs[qid].ready = false;
+ pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR;
+ pdsv->vqs[qid].notify =
+ vp_modern_map_vq_notify(>vdpa_aux->vd_mdev,
+ qid, >vqs[qid].notify_pa);
Nit: It looks to me this would not change. So we probably don't need
this during reset?
We set it again here because we used memset to clean the struct and need
to put it back. But we could grap the value before the memset then
restore it, and do the map_vq_notify call just the first time. I'll fix
that up for v2.
sln
Thanks
+}
+
static int pds_vdpa_reset(struct vdpa_device *vdpa_dev)
{
struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
@@ -451,8 +463,7 @@ static int pds_vdpa_reset(struct vdpa_device *vdpa_dev)
dev_err(dev, "%s: reset_vq failed qid %d:
%pe\n",
__func__, i, ERR_PTR(err));
pds_vdpa_release_irq(pdsv, i);
- memset(>vqs[i], 0, sizeof(pdsv->vqs[0]));
- pdsv->vqs[i].ready = false;
+ pds_vdpa_init_vqs_entry(pdsv, i);
}
}
@@ -640,13 +651,8 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
}
pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
- for (i = 0; i < pdsv->num_vqs; i++) {
- pdsv->vqs[i].qid = i;
- pdsv->vqs[i].pdsv = pdsv;
- pdsv->vqs[i].irq = VIRTIO_MSI_NO_VECTOR;
- pdsv->vqs[i].notify =
vp_modern_map_vq_notify(>vdpa_aux->vd_mdev,
- i,
>vqs[i].notify_pa);
- }
+ for (i = 0; i < pdsv->num_vqs; i++)
+ pds_vdpa_init_vqs_entry(pdsv, i);
pdsv->vdpa_dev.mdev = _aux->vdpa_mdev;
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH virtio 2/4] pds_vdpa: always allow offering VIRTIO_NET_F_MAC
On 7/7/23 12:44 AM, Jason Wang wrote:
On Fri, Jun 30, 2023 at 8:36 AM Shannon Nelson wrote:
Our driver sets a mac if the HW is 00:..:00 so we need to be sure to
advertise VIRTIO_NET_F_MAC even if the HW doesn't. We also need to be
sure that virtio_net sees the VIRTIO_NET_F_MAC and doesn't rewrite the
mac address that a user may have set with the vdpa utility.
After reading the hw_feature bits, add the VIRTIO_NET_F_MAC to the driver's
supported_features and use that for reporting what is available. If the
HW is not advertising it, be sure to strip the VIRTIO_NET_F_MAC before
finishing the feature negotiation. If the user specifies a device_features
bitpattern in the vdpa utility without the VIRTIO_NET_F_MAC set, then
don't set the mac.
Fixes: 151cc834f3dd ("pds_vdpa: add support for vdpa and vdpamgmt interfaces")
Signed-off-by: Shannon Nelson
Reviewed-by: Brett Creeley
---
drivers/vdpa/pds/vdpa_dev.c | 25 +++--
1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index e2e99bb0be2b..5e761d625ef3 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -316,6 +316,7 @@ static int pds_vdpa_set_driver_features(struct vdpa_device
*vdpa_dev, u64 featur
{
struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
struct device *dev = >vdpa_dev.dev;
+ u64 requested_features;
u64 driver_features;
u64 nego_features;
u64 missing;
@@ -325,18 +326,24 @@ static int pds_vdpa_set_driver_features(struct
vdpa_device *vdpa_dev, u64 featur
return -EOPNOTSUPP;
}
+ /* save original request for debugfs */
pdsv->req_features = features;
+ requested_features = features;
+
+ /* if we're faking the F_MAC, strip it from features to be negotiated */
+ driver_features = pds_vdpa_get_driver_features(vdpa_dev);
+ if (!(driver_features & BIT_ULL(VIRTIO_NET_F_MAC)))
+ requested_features &= ~BIT_ULL(VIRTIO_NET_F_MAC);
I'm not sure I understand here, assuming we are doing feature
negotiation here. In this case driver_features we read should be zero?
Or did you actually mean device_features here?
Yes, this needs to be cleared up. I'll address it in v2.
sln
Thanks
/* Check for valid feature bits */
- nego_features = features &
le64_to_cpu(pdsv->vdpa_aux->ident.hw_features);
- missing = pdsv->req_features & ~nego_features;
+ nego_features = requested_features &
le64_to_cpu(pdsv->vdpa_aux->ident.hw_features);
+ missing = requested_features & ~nego_features;
if (missing) {
dev_err(dev, "Can't support all requested features in %#llx,
missing %#llx features\n",
pdsv->req_features, missing);
return -EOPNOTSUPP;
}
- driver_features = pds_vdpa_get_driver_features(vdpa_dev);
dev_dbg(dev, "%s: %#llx => %#llx\n",
__func__, driver_features, nego_features);
@@ -564,7 +571,7 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
u64 unsupp_features =
- add_config->device_features & ~mgmt->supported_features;
+ add_config->device_features & ~pdsv->supported_features;
if (unsupp_features) {
dev_err(dev, "Unsupported features: %#llx\n",
unsupp_features);
@@ -615,7 +622,8 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
}
/* Set a mac, either from the user config if provided
-* or set a random mac if default is 00:..:00
+* or use the device's mac if not 00:..:00
+* or set a random mac
*/
if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
ether_addr_copy(pdsv->mac, add_config->net.mac);
@@ -624,7 +632,8 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
vc = pdsv->vdpa_aux->vd_mdev.device;
memcpy_fromio(pdsv->mac, vc->mac, sizeof(pdsv->mac));
- if (is_zero_ether_addr(pdsv->mac)) {
+ if (is_zero_ether_addr(pdsv->mac) &&
+ (pdsv->supported_features & BIT_ULL(VIRTIO_NET_F_MAC))) {
eth_random_addr(pdsv->mac);
dev_info(dev, "setting random mac %pM\n", pdsv->mac);
}
@@ -752,6 +761,10 @@ int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
mgmt->id_table = pds_vdpa_id_table;
mgmt->device = dev;
mgmt->supported_features = le64_to_cpu(vdpa_aux->ident.hw_features);
+
+ /* advertise F_MAC even if the device doesn't */
+ mgmt->supported_features |= BIT_ULL(VIRTIO_NET_F_MAC);
+
mgmt->config_attr_mask =
Re: [PATCH virtio 1/4] pds_vdpa: reset to vdpa specified mac
On 7/7/23 12:33 AM, Jason Wang wrote:
Caution: This message originated from an External Source. Use proper caution
when opening attachments, clicking links, or responding.
On Fri, Jun 30, 2023 at 8:36 AM Shannon Nelson wrote:
From: Allen Hubbe
When the vdpa device is reset, also reinitialize it with the mac address
that was assigned when the device was added.
Fixes: 151cc834f3dd ("pds_vdpa: add support for vdpa and vdpamgmt interfaces")
Signed-off-by: Allen Hubbe
Signed-off-by: Shannon Nelson
Reviewed-by: Brett Creeley
---
drivers/vdpa/pds/vdpa_dev.c | 16
drivers/vdpa/pds/vdpa_dev.h | 1 +
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index 5071a4d58f8d..e2e99bb0be2b 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -409,6 +409,8 @@ static void pds_vdpa_set_status(struct vdpa_device
*vdpa_dev, u8 status)
pdsv->vqs[i].avail_idx = 0;
pdsv->vqs[i].used_idx = 0;
}
+
+ pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
So this is not necessarily called during reset. So I think we need to
move it to pds_vdpa_reset()?
pds_vdpa_reset() calls pds_vdpa_set_status() with a status=0, so this is
already covered.
sln
The rest looks good.
Thanks
}
if (status & ~old_status & VIRTIO_CONFIG_S_FEATURES_OK) {
@@ -532,7 +534,6 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
struct device *dma_dev;
struct pci_dev *pdev;
struct device *dev;
- u8 mac[ETH_ALEN];
int err;
int i;
@@ -617,19 +618,18 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
* or set a random mac if default is 00:..:00
*/
if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
- ether_addr_copy(mac, add_config->net.mac);
- pds_vdpa_cmd_set_mac(pdsv, mac);
+ ether_addr_copy(pdsv->mac, add_config->net.mac);
} else {
struct virtio_net_config __iomem *vc;
vc = pdsv->vdpa_aux->vd_mdev.device;
- memcpy_fromio(mac, vc->mac, sizeof(mac));
- if (is_zero_ether_addr(mac)) {
- eth_random_addr(mac);
- dev_info(dev, "setting random mac %pM\n", mac);
- pds_vdpa_cmd_set_mac(pdsv, mac);
+ memcpy_fromio(pdsv->mac, vc->mac, sizeof(pdsv->mac));
+ if (is_zero_ether_addr(pdsv->mac)) {
+ eth_random_addr(pdsv->mac);
+ dev_info(dev, "setting random mac %pM\n", pdsv->mac);
}
}
+ pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
for (i = 0; i < pdsv->num_vqs; i++) {
pdsv->vqs[i].qid = i;
diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h
index a1bc37de9537..cf02df287fc4 100644
--- a/drivers/vdpa/pds/vdpa_dev.h
+++ b/drivers/vdpa/pds/vdpa_dev.h
@@ -39,6 +39,7 @@ struct pds_vdpa_device {
u64 req_features; /* features requested by vdpa */
u8 vdpa_index; /* rsvd for future subdevice use */
u8 num_vqs; /* num vqs in use */
+ u8 mac[ETH_ALEN]; /* mac selected when the device was
added */
struct vdpa_callback config_cb;
struct notifier_block nb;
};
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH virtio] pds_vdpa: protect Makefile from unconfigured debugfs
debugfs.h protects itself from an undefined DEBUG_FS, so it is
not necessary to check it in the driver code or the Makefile.
The driver code had been updated for this, but the Makefile had
missed the update.
Link:
https://lore.kernel.org/linux-next/fec68c3c-8249-7af4-5390-0495386a7...@infradead.org/
Fixes: a16291b5bcbb ("pds_vdpa: Add new vDPA driver for AMD/Pensando DSC")
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/Makefile | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
index 2e22418e3ab3..c2d314d4614d 100644
--- a/drivers/vdpa/pds/Makefile
+++ b/drivers/vdpa/pds/Makefile
@@ -5,6 +5,5 @@ obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
pds_vdpa-y := aux_drv.o \
cmds.o \
+ debugfs.o \
vdpa_dev.o
-
-pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: linux-next: Tree for Jul 6 [drivers/vdpa/pds/pds_vdpa.ko]
On 7/6/23 2:07 PM, Randy Dunlap wrote: On 7/5/23 18:57, Stephen Rothwell wrote: Hi all, Please do *not* add any v6.6 related stuff to your linux-next included branches until after v6.5-rc1 has been released. Changes since 20230705: On loongarch, when DEBUG_FS is not set: ERROR: modpost: "pds_vdpa_debugfs_del_vdpadev" [drivers/vdpa/pds/pds_vdpa.ko] undefined! ERROR: modpost: "pds_vdpa_debugfs_add_vdpadev" [drivers/vdpa/pds/pds_vdpa.ko] undefined! ERROR: modpost: "pds_vdpa_debugfs_reset_vdpadev" [drivers/vdpa/pds/pds_vdpa.ko] undefined! ERROR: modpost: "pds_vdpa_debugfs_create" [drivers/vdpa/pds/pds_vdpa.ko] undefined! ERROR: modpost: "pds_vdpa_debugfs_add_ident" [drivers/vdpa/pds/pds_vdpa.ko] undefined! ERROR: modpost: "pds_vdpa_debugfs_destroy" [drivers/vdpa/pds/pds_vdpa.ko] undefined! ERROR: modpost: "pds_vdpa_debugfs_add_pcidev" [drivers/vdpa/pds/pds_vdpa.ko] undefined! Yikes - sorry, will follow up shortly. sln Full randconfig file is attached. -- ~Randy ___ Virtualization mailing list Virtualization@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH] vdpa: reject F_ENABLE_AFTER_DRIVER_OK if backend does not support it
On 7/5/23 11:27 AM, Eugenio Perez Martin wrote:
>
On Wed, Jul 5, 2023 at 9:50 AM Jason Wang wrote:
On Tue, Jul 4, 2023 at 11:45 PM Michael S. Tsirkin wrote:
On Tue, Jul 04, 2023 at 01:36:11PM +0200, Eugenio Perez Martin wrote:
On Tue, Jul 4, 2023 at 12:38 PM Michael S. Tsirkin wrote:
On Tue, Jul 04, 2023 at 12:25:32PM +0200, Eugenio Perez Martin wrote:
On Mon, Jul 3, 2023 at 4:52 PM Michael S. Tsirkin wrote:
On Mon, Jul 03, 2023 at 04:22:18PM +0200, Eugenio Pérez wrote:
With the current code it is accepted as long as userland send it.
Although userland should not set a feature flag that has not been
offered to it with VHOST_GET_BACKEND_FEATURES, the current code will not
complain for it.
Since there is no specific reason for any parent to reject that backend
feature bit when it has been proposed, let's control it at vdpa frontend
level. Future patches may move this control to the parent driver.
Fixes: 967800d2d52e ("vdpa: accept VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK backend
feature")
Signed-off-by: Eugenio Pérez
Please do send v3. And again, I don't want to send "after driver ok" hack
upstream at all, I merged it in next just to give it some testing.
We want RING_ACCESS_AFTER_KICK or some such.
Current devices do not support that semantic.
Which devices specifically access the ring after DRIVER_OK but before
a kick?
The PDS vdpa device can deal with a call to .set_vq_ready after
DRIVER_OK is set. And I'm told that our VQ activity should start
without a kick.
Our vdpa device FW doesn't currently have support for
VIRTIO_F_RING_RESET, but I believe it could be added without too much
trouble.
sln
Previous versions of the QEMU LM series did a spurious kick to start
traffic at the LM destination [1]. When it was proposed, that spurious
kick was removed from the series because to check for descriptors
after driver_ok, even without a kick, was considered work of the
parent driver.
I'm ok to go back to this spurious kick, but I'm not sure if the hw
will read the ring before the kick actually. I can ask.
Thanks!
[1] https://lists.nongnu.org/archive/html/qemu-devel/2023-01/msg02775.html
Let's find out. We need to check for ENABLE_AFTER_DRIVER_OK too, no?
My understanding is [1] assuming ACCESS_AFTER_KICK. This seems
sub-optimal than assuming ENABLE_AFTER_DRIVER_OK.
But this reminds me one thing, as the thread is going too long, I
wonder if we simply assume ENABLE_AFTER_DRIVER_OK if RING_RESET is
supported?
The problem with that is that the device needs to support all
RING_RESET, like to be able to change vq address etc after DRIVER_OK.
Not all HW support it.
We just need the subset of having the dataplane freezed until all CVQ
commands have been consumed. I'm sure current vDPA code already
supports it in some devices, like MLX and PSD.
Thanks!
Thanks
My plan was to convert
it in vp_vdpa if needed, and reuse the current vdpa ops. Sorry if I
was not explicit enough.
The only solution I can see to that is to trap & emulate in the vdpa
(parent?) driver, as talked in virtio-comment. But that complicates
the architecture:
* Offer VHOST_BACKEND_F_RING_ACCESS_AFTER_KICK
* Store vq enable state separately, at
vdpa->config->set_vq_ready(true), but not transmit that enable to hw
* Store the doorbell state separately, but do not configure it to the
device directly.
But how to recover if the device cannot configure them at kick time,
for example?
Maybe we can just fail if the parent driver does not support enabling
the vq after DRIVER_OK? That way no new feature flag is needed.
Thanks!
---
Sent with Fixes: tag pointing to git.kernel.org/pub/scm/linux/kernel/git/mst
commit. Please let me know if I should send a v3 of [1] instead.
[1]
https://lore.kernel.org/lkml/20230609121244-mutt-send-email-...@kernel.org/T/
---
drivers/vhost/vdpa.c | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index e1abf29fed5b..a7e554352351 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -681,18 +681,21 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep,
{
struct vhost_vdpa *v = filep->private_data;
struct vhost_dev *d = >vdev;
+ const struct vdpa_config_ops *ops = v->vdpa->config;
void __user *argp = (void __user *)arg;
u64 __user *featurep = argp;
- u64 features;
+ u64 features, parent_features = 0;
long r = 0;
if (cmd == VHOST_SET_BACKEND_FEATURES) {
if (copy_from_user(, featurep, sizeof(features)))
return -EFAULT;
+ if (ops->get_backend_features)
+ parent_features = ops->get_backend_features(v->vdpa);
if (features & ~(VHOST_VDPA_BACKEND_FEATURES |
BIT_ULL(VHOST_BACKEND_F_SUSPEND) |
BIT_ULL(VHOST_BACKEND_F_RESUME) |
-
[PATCH virtio 1/4] pds_vdpa: reset to vdpa specified mac
From: Allen Hubbe
When the vdpa device is reset, also reinitialize it with the mac address
that was assigned when the device was added.
Fixes: 151cc834f3dd ("pds_vdpa: add support for vdpa and vdpamgmt interfaces")
Signed-off-by: Allen Hubbe
Signed-off-by: Shannon Nelson
Reviewed-by: Brett Creeley
---
drivers/vdpa/pds/vdpa_dev.c | 16
drivers/vdpa/pds/vdpa_dev.h | 1 +
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index 5071a4d58f8d..e2e99bb0be2b 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -409,6 +409,8 @@ static void pds_vdpa_set_status(struct vdpa_device
*vdpa_dev, u8 status)
pdsv->vqs[i].avail_idx = 0;
pdsv->vqs[i].used_idx = 0;
}
+
+ pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
}
if (status & ~old_status & VIRTIO_CONFIG_S_FEATURES_OK) {
@@ -532,7 +534,6 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
struct device *dma_dev;
struct pci_dev *pdev;
struct device *dev;
- u8 mac[ETH_ALEN];
int err;
int i;
@@ -617,19 +618,18 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
* or set a random mac if default is 00:..:00
*/
if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
- ether_addr_copy(mac, add_config->net.mac);
- pds_vdpa_cmd_set_mac(pdsv, mac);
+ ether_addr_copy(pdsv->mac, add_config->net.mac);
} else {
struct virtio_net_config __iomem *vc;
vc = pdsv->vdpa_aux->vd_mdev.device;
- memcpy_fromio(mac, vc->mac, sizeof(mac));
- if (is_zero_ether_addr(mac)) {
- eth_random_addr(mac);
- dev_info(dev, "setting random mac %pM\n", mac);
- pds_vdpa_cmd_set_mac(pdsv, mac);
+ memcpy_fromio(pdsv->mac, vc->mac, sizeof(pdsv->mac));
+ if (is_zero_ether_addr(pdsv->mac)) {
+ eth_random_addr(pdsv->mac);
+ dev_info(dev, "setting random mac %pM\n", pdsv->mac);
}
}
+ pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
for (i = 0; i < pdsv->num_vqs; i++) {
pdsv->vqs[i].qid = i;
diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h
index a1bc37de9537..cf02df287fc4 100644
--- a/drivers/vdpa/pds/vdpa_dev.h
+++ b/drivers/vdpa/pds/vdpa_dev.h
@@ -39,6 +39,7 @@ struct pds_vdpa_device {
u64 req_features; /* features requested by vdpa */
u8 vdpa_index; /* rsvd for future subdevice use */
u8 num_vqs; /* num vqs in use */
+ u8 mac[ETH_ALEN]; /* mac selected when the device was
added */
struct vdpa_callback config_cb;
struct notifier_block nb;
};
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH virtio 2/4] pds_vdpa: always allow offering VIRTIO_NET_F_MAC
Our driver sets a mac if the HW is 00:..:00 so we need to be sure to
advertise VIRTIO_NET_F_MAC even if the HW doesn't. We also need to be
sure that virtio_net sees the VIRTIO_NET_F_MAC and doesn't rewrite the
mac address that a user may have set with the vdpa utility.
After reading the hw_feature bits, add the VIRTIO_NET_F_MAC to the driver's
supported_features and use that for reporting what is available. If the
HW is not advertising it, be sure to strip the VIRTIO_NET_F_MAC before
finishing the feature negotiation. If the user specifies a device_features
bitpattern in the vdpa utility without the VIRTIO_NET_F_MAC set, then
don't set the mac.
Fixes: 151cc834f3dd ("pds_vdpa: add support for vdpa and vdpamgmt interfaces")
Signed-off-by: Shannon Nelson
Reviewed-by: Brett Creeley
---
drivers/vdpa/pds/vdpa_dev.c | 25 +++--
1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index e2e99bb0be2b..5e761d625ef3 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -316,6 +316,7 @@ static int pds_vdpa_set_driver_features(struct vdpa_device
*vdpa_dev, u64 featur
{
struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
struct device *dev = >vdpa_dev.dev;
+ u64 requested_features;
u64 driver_features;
u64 nego_features;
u64 missing;
@@ -325,18 +326,24 @@ static int pds_vdpa_set_driver_features(struct
vdpa_device *vdpa_dev, u64 featur
return -EOPNOTSUPP;
}
+ /* save original request for debugfs */
pdsv->req_features = features;
+ requested_features = features;
+
+ /* if we're faking the F_MAC, strip it from features to be negotiated */
+ driver_features = pds_vdpa_get_driver_features(vdpa_dev);
+ if (!(driver_features & BIT_ULL(VIRTIO_NET_F_MAC)))
+ requested_features &= ~BIT_ULL(VIRTIO_NET_F_MAC);
/* Check for valid feature bits */
- nego_features = features &
le64_to_cpu(pdsv->vdpa_aux->ident.hw_features);
- missing = pdsv->req_features & ~nego_features;
+ nego_features = requested_features &
le64_to_cpu(pdsv->vdpa_aux->ident.hw_features);
+ missing = requested_features & ~nego_features;
if (missing) {
dev_err(dev, "Can't support all requested features in %#llx,
missing %#llx features\n",
pdsv->req_features, missing);
return -EOPNOTSUPP;
}
- driver_features = pds_vdpa_get_driver_features(vdpa_dev);
dev_dbg(dev, "%s: %#llx => %#llx\n",
__func__, driver_features, nego_features);
@@ -564,7 +571,7 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
u64 unsupp_features =
- add_config->device_features & ~mgmt->supported_features;
+ add_config->device_features & ~pdsv->supported_features;
if (unsupp_features) {
dev_err(dev, "Unsupported features: %#llx\n",
unsupp_features);
@@ -615,7 +622,8 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
}
/* Set a mac, either from the user config if provided
-* or set a random mac if default is 00:..:00
+* or use the device's mac if not 00:..:00
+* or set a random mac
*/
if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
ether_addr_copy(pdsv->mac, add_config->net.mac);
@@ -624,7 +632,8 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
vc = pdsv->vdpa_aux->vd_mdev.device;
memcpy_fromio(pdsv->mac, vc->mac, sizeof(pdsv->mac));
- if (is_zero_ether_addr(pdsv->mac)) {
+ if (is_zero_ether_addr(pdsv->mac) &&
+ (pdsv->supported_features & BIT_ULL(VIRTIO_NET_F_MAC))) {
eth_random_addr(pdsv->mac);
dev_info(dev, "setting random mac %pM\n", pdsv->mac);
}
@@ -752,6 +761,10 @@ int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
mgmt->id_table = pds_vdpa_id_table;
mgmt->device = dev;
mgmt->supported_features = le64_to_cpu(vdpa_aux->ident.hw_features);
+
+ /* advertise F_MAC even if the device doesn't */
+ mgmt->supported_features |= BIT_ULL(VIRTIO_NET_F_MAC);
+
mgmt->config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR);
mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);
mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_FEATURES);
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH virtio 0/4] pds_vdpa: mac, reset, and irq updates
v2 for internal review - heavily reworked NET_F_MAC patch, matches recent PR-68875 - reordered to put "clean and reset vqs" before "alloc-irq" to make them slightly simpler patches - other minor cleanups for simpler patches These are some fixes for device providing a MAC address, for allocating irq resources later to support vhost use, and for properly cleaning vq info on reset. Allen Hubbe (2): pds_vdpa: reset to vdpa specified mac pds_vdpa: alloc irq vectors on DRIVER_OK Shannon Nelson (2): pds_vdpa: always allow offering VIRTIO_NET_F_MAC pds_vdpa: clean and reset vqs entries drivers/vdpa/pds/vdpa_dev.c | 171 +--- drivers/vdpa/pds/vdpa_dev.h | 1 + 2 files changed, 122 insertions(+), 50 deletions(-) -- 2.17.1 ___ Virtualization mailing list Virtualization@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH virtio 3/4] pds_vdpa: clean and reset vqs entries
Make sure that we initialize the vqs[] entries the same
way both for initial setup and after a vq reset.
Fixes: 151cc834f3dd ("pds_vdpa: add support for vdpa and vdpamgmt interfaces")
Signed-off-by: Shannon Nelson
Reviewed-by: Brett Creeley
---
drivers/vdpa/pds/vdpa_dev.c | 24 +++-
1 file changed, 15 insertions(+), 9 deletions(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index 5e761d625ef3..5e1046c9af3d 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -429,6 +429,18 @@ static void pds_vdpa_set_status(struct vdpa_device
*vdpa_dev, u8 status)
}
}
+static void pds_vdpa_init_vqs_entry(struct pds_vdpa_device *pdsv, int qid)
+{
+ memset(>vqs[qid], 0, sizeof(pdsv->vqs[0]));
+ pdsv->vqs[qid].qid = qid;
+ pdsv->vqs[qid].pdsv = pdsv;
+ pdsv->vqs[qid].ready = false;
+ pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR;
+ pdsv->vqs[qid].notify =
+ vp_modern_map_vq_notify(>vdpa_aux->vd_mdev,
+ qid, >vqs[qid].notify_pa);
+}
+
static int pds_vdpa_reset(struct vdpa_device *vdpa_dev)
{
struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
@@ -451,8 +463,7 @@ static int pds_vdpa_reset(struct vdpa_device *vdpa_dev)
dev_err(dev, "%s: reset_vq failed qid %d:
%pe\n",
__func__, i, ERR_PTR(err));
pds_vdpa_release_irq(pdsv, i);
- memset(>vqs[i], 0, sizeof(pdsv->vqs[0]));
- pdsv->vqs[i].ready = false;
+ pds_vdpa_init_vqs_entry(pdsv, i);
}
}
@@ -640,13 +651,8 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
}
pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
- for (i = 0; i < pdsv->num_vqs; i++) {
- pdsv->vqs[i].qid = i;
- pdsv->vqs[i].pdsv = pdsv;
- pdsv->vqs[i].irq = VIRTIO_MSI_NO_VECTOR;
- pdsv->vqs[i].notify =
vp_modern_map_vq_notify(>vdpa_aux->vd_mdev,
- i,
>vqs[i].notify_pa);
- }
+ for (i = 0; i < pdsv->num_vqs; i++)
+ pds_vdpa_init_vqs_entry(pdsv, i);
pdsv->vdpa_dev.mdev = _aux->vdpa_mdev;
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH virtio 4/4] pds_vdpa: alloc irq vectors on DRIVER_OK
From: Allen Hubbe
We were allocating irq vectors at the time the aux dev was probed,
but that is before the PCI VF is assigned to a separate iommu domain
by vhost_vdpa. Because vhost_vdpa later changes the iommu domain the
interrupts do not work.
Instead, we can allocate the irq vectors later when we see DRIVER_OK and
know that the reassignment of the PCI VF to an iommu domain has already
happened.
Fixes: 151cc834f3dd ("pds_vdpa: add support for vdpa and vdpamgmt interfaces")
Signed-off-by: Allen Hubbe
Signed-off-by: Shannon Nelson
Reviewed-by: Brett Creeley
---
drivers/vdpa/pds/vdpa_dev.c | 110 ++--
1 file changed, 81 insertions(+), 29 deletions(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index 5e1046c9af3d..6c337f7a0f06 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -126,11 +126,9 @@ static void pds_vdpa_release_irq(struct pds_vdpa_device
*pdsv, int qid)
static void pds_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool
ready)
{
struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
- struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;
struct device *dev = >vdpa_dev.dev;
u64 driver_features;
u16 invert_idx = 0;
- int irq;
int err;
dev_dbg(dev, "%s: qid %d ready %d => %d\n",
@@ -143,19 +141,6 @@ static void pds_vdpa_set_vq_ready(struct vdpa_device
*vdpa_dev, u16 qid, bool re
invert_idx = PDS_VDPA_PACKED_INVERT_IDX;
if (ready) {
- irq = pci_irq_vector(pdev, qid);
- snprintf(pdsv->vqs[qid].irq_name,
sizeof(pdsv->vqs[qid].irq_name),
-"vdpa-%s-%d", dev_name(dev), qid);
-
- err = request_irq(irq, pds_vdpa_isr, 0,
- pdsv->vqs[qid].irq_name, >vqs[qid]);
- if (err) {
- dev_err(dev, "%s: no irq for qid %d: %pe\n",
- __func__, qid, ERR_PTR(err));
- return;
- }
- pdsv->vqs[qid].irq = irq;
-
/* Pass vq setup info to DSC using adminq to gather up and
* send all info at once so FW can do its full set up in
* one easy operation
@@ -164,7 +149,6 @@ static void pds_vdpa_set_vq_ready(struct vdpa_device
*vdpa_dev, u16 qid, bool re
if (err) {
dev_err(dev, "Failed to init vq %d: %pe\n",
qid, ERR_PTR(err));
- pds_vdpa_release_irq(pdsv, qid);
ready = false;
}
} else {
@@ -172,7 +156,6 @@ static void pds_vdpa_set_vq_ready(struct vdpa_device
*vdpa_dev, u16 qid, bool re
if (err)
dev_err(dev, "%s: reset_vq failed qid %d: %pe\n",
__func__, qid, ERR_PTR(err));
- pds_vdpa_release_irq(pdsv, qid);
}
pdsv->vqs[qid].ready = ready;
@@ -396,6 +379,72 @@ static u8 pds_vdpa_get_status(struct vdpa_device *vdpa_dev)
return vp_modern_get_status(>vdpa_aux->vd_mdev);
}
+static int pds_vdpa_request_irqs(struct pds_vdpa_device *pdsv)
+{
+ struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;
+ struct pds_vdpa_aux *vdpa_aux = pdsv->vdpa_aux;
+ struct device *dev = >vdpa_dev.dev;
+ int max_vq, nintrs, qid, err;
+
+ max_vq = vdpa_aux->vdpa_mdev.max_supported_vqs;
+
+ nintrs = pci_alloc_irq_vectors(pdev, max_vq, max_vq, PCI_IRQ_MSIX);
+ if (nintrs < 0) {
+ dev_err(dev, "Couldn't get %d msix vectors: %pe\n",
+ max_vq, ERR_PTR(nintrs));
+ return nintrs;
+ }
+
+ for (qid = 0; qid < pdsv->num_vqs; ++qid) {
+ int irq = pci_irq_vector(pdev, qid);
+
+ snprintf(pdsv->vqs[qid].irq_name,
sizeof(pdsv->vqs[qid].irq_name),
+"vdpa-%s-%d", dev_name(dev), qid);
+
+ err = request_irq(irq, pds_vdpa_isr, 0,
+ pdsv->vqs[qid].irq_name,
+ >vqs[qid]);
+ if (err) {
+ dev_err(dev, "%s: no irq for qid %d: %pe\n",
+ __func__, qid, ERR_PTR(err));
+ goto err_release;
+ }
+
+ pdsv->vqs[qid].irq = irq;
+ }
+
+ vdpa_aux->nintrs = nintrs;
+
+ return 0;
+
+err_release:
+ while (qid--)
+ pds_vdpa_release_irq(pdsv, qid);
+
+ pci_free_irq_vectors(pdev);
+
+ vdpa_aux->nintrs = 0;
+
+ return err;
+}
+
+static void pds_vdpa_release_irqs(struct pds_vdpa_device *pdsv)
+{
+ struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;
+ struct pds_vdpa_aux *vdpa_aux = pdsv->vdpa_aux;
+ int qid;
+
+ if (!vdpa_aux->nintrs)
+ return;
+
+
[PATCH v7 virtio 11/11] pds_vdpa: pds_vdps.rst and Kconfig
Add the documentation and Kconfig entry for pds_vdpa driver.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
.../device_drivers/ethernet/amd/pds_vdpa.rst | 85 +++
.../device_drivers/ethernet/index.rst | 1 +
MAINTAINERS | 4 +
drivers/vdpa/Kconfig | 10 +++
4 files changed, 100 insertions(+)
create mode 100644
Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
diff --git a/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
b/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
new file mode 100644
index ..587927d3de92
--- /dev/null
+++ b/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
@@ -0,0 +1,85 @@
+.. SPDX-License-Identifier: GPL-2.0+
+.. note: can be edited and viewed with /usr/bin/formiko-vim
+
+==
+PCI vDPA driver for the AMD/Pensando(R) DSC adapter family
+==
+
+AMD/Pensando vDPA VF Device Driver
+
+Copyright(c) 2023 Advanced Micro Devices, Inc
+
+Overview
+
+
+The ``pds_vdpa`` driver is an auxiliary bus driver that supplies
+a vDPA device for use by the virtio network stack. It is used with
+the Pensando Virtual Function devices that offer vDPA and virtio queue
+services. It depends on the ``pds_core`` driver and hardware for the PF
+and VF PCI handling as well as for device configuration services.
+
+Using the device
+
+
+The ``pds_vdpa`` device is enabled via multiple configuration steps and
+depends on the ``pds_core`` driver to create and enable SR-IOV Virtual
+Function devices. After the VFs are enabled, we enable the vDPA service
+in the ``pds_core`` device to create the auxiliary devices used by pds_vdpa.
+
+Example steps:
+
+.. code-block:: bash
+
+ #!/bin/bash
+
+ modprobe pds_core
+ modprobe vdpa
+ modprobe pds_vdpa
+
+ PF_BDF=`ls /sys/module/pds_core/drivers/pci\:pds_core/*/sriov_numvfs | awk
-F / '{print $7}'`
+
+ # Enable vDPA VF auxiliary device(s) in the PF
+ devlink dev param set pci/$PF_BDF name enable_vnet cmode runtime value true
+
+ # Create a VF for vDPA use
+ echo 1 > /sys/bus/pci/drivers/pds_core/$PF_BDF/sriov_numvfs
+
+ # Find the vDPA services/devices available
+ PDS_VDPA_MGMT=`vdpa mgmtdev show | grep vDPA | head -1 | cut -d: -f1`
+
+ # Create a vDPA device for use in virtio network configurations
+ vdpa dev add name vdpa1 mgmtdev $PDS_VDPA_MGMT mac 00:11:22:33:44:55
+
+ # Set up an ethernet interface on the vdpa device
+ modprobe virtio_vdpa
+
+
+
+Enabling the driver
+===
+
+The driver is enabled via the standard kernel configuration system,
+using the make command::
+
+ make oldconfig/menuconfig/etc.
+
+The driver is located in the menu structure at:
+
+ -> Device Drivers
+-> Network device support (NETDEVICES [=y])
+ -> Ethernet driver support
+-> Pensando devices
+ -> Pensando Ethernet PDS_VDPA Support
+
+Support
+===
+
+For general Linux networking support, please use the netdev mailing
+list, which is monitored by Pensando personnel::
+
+ net...@vger.kernel.org
+
+For more specific support needs, please use the Pensando driver support
+email::
+
+ driv...@pensando.io
diff --git a/Documentation/networking/device_drivers/ethernet/index.rst
b/Documentation/networking/device_drivers/ethernet/index.rst
index 417ca514a4d0..94ecb67c0885 100644
--- a/Documentation/networking/device_drivers/ethernet/index.rst
+++ b/Documentation/networking/device_drivers/ethernet/index.rst
@@ -15,6 +15,7 @@ Contents:
amazon/ena
altera/altera_tse
amd/pds_core
+ amd/pds_vdpa
aquantia/atlantic
chelsio/cxgb
cirrus/cs89x0
diff --git a/MAINTAINERS b/MAINTAINERS
index c25172d6471a..2bd6d0e8a084 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -22304,6 +22304,10 @@ F: include/linux/vringh.h
F: include/uapi/linux/virtio_*.h
F: tools/virtio/
+PDS DSC VIRTIO DATA PATH ACCELERATOR
+R: Shannon Nelson
+F: drivers/vdpa/pds/
+
VIRTIO CRYPTO DRIVER
M: Gonglei
L: virtualization@lists.linux-foundation.org
diff --git a/drivers/vdpa/Kconfig b/drivers/vdpa/Kconfig
index cd6ad92f3f05..656c1cb541de 100644
--- a/drivers/vdpa/Kconfig
+++ b/drivers/vdpa/Kconfig
@@ -116,4 +116,14 @@ config ALIBABA_ENI_VDPA
This driver includes a HW monitor device that
reads health values from the DPU.
+config PDS_VDPA
+ tristate "vDPA driver for AMD/Pensando DSC devices"
+ select VIRTIO_PCI_LIB
+ depends on PCI_MSI
+ depends on PDS_CORE
+ help
+ vDPA network driver for AMD/Pensando's PDS Core devices.
+ With this driver, the VirtIO dataplane can be
+ offloaded to an AMD/Pensando DSC device.
+
endif # VDPA
--
2.17.1
___
Virtualization mailing list
[PATCH v7 virtio 08/11] pds_vdpa: add vdpa config client commands
These are the adminq commands that will be needed for
setting up and using the vDPA device. There are a number
of commands defined in the FW's API, but by making use of
the FW's virtio BAR we only need a few of these commands
for vDPA support.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/pds/Makefile | 1 +
drivers/vdpa/pds/cmds.c | 185
drivers/vdpa/pds/cmds.h | 18
drivers/vdpa/pds/vdpa_dev.h | 33 ++-
4 files changed, 236 insertions(+), 1 deletion(-)
create mode 100644 drivers/vdpa/pds/cmds.c
create mode 100644 drivers/vdpa/pds/cmds.h
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
index 13b50394ec64..2e22418e3ab3 100644
--- a/drivers/vdpa/pds/Makefile
+++ b/drivers/vdpa/pds/Makefile
@@ -4,6 +4,7 @@
obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
pds_vdpa-y := aux_drv.o \
+ cmds.o \
vdpa_dev.o
pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/vdpa/pds/cmds.c b/drivers/vdpa/pds/cmds.c
new file mode 100644
index ..80863a41c3cd
--- /dev/null
+++ b/drivers/vdpa/pds/cmds.c
@@ -0,0 +1,185 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include "vdpa_dev.h"
+#include "aux_drv.h"
+#include "cmds.h"
+
+int pds_vdpa_init_hw(struct pds_vdpa_device *pdsv)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_init.opcode = PDS_VDPA_CMD_INIT,
+ .vdpa_init.vdpa_index = pdsv->vdpa_index,
+ .vdpa_init.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ /* Initialize the vdpa/virtio device */
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa_init),
+ , 0);
+ if (err)
+ dev_dbg(dev, "Failed to init hw, status %d: %pe\n",
+ comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_reset(struct pds_vdpa_device *pdsv)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa.opcode = PDS_VDPA_CMD_RESET,
+ .vdpa.vdpa_index = pdsv->vdpa_index,
+ .vdpa.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa), , 0);
+ if (err)
+ dev_dbg(dev, "Failed to reset hw, status %d: %pe\n",
+ comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_set_status(struct pds_vdpa_device *pdsv, u8 status)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_status.opcode = PDS_VDPA_CMD_STATUS_UPDATE,
+ .vdpa_status.vdpa_index = pdsv->vdpa_index,
+ .vdpa_status.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ .vdpa_status.status = status,
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa_status),
, 0);
+ if (err)
+ dev_dbg(dev, "Failed to set status to %#x, error status %d:
%pe\n",
+ status, comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_set_mac(struct pds_vdpa_device *pdsv, u8 *mac)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR,
+ .vdpa_setattr.vdpa_index = pdsv->vdpa_index,
+ .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ .vdpa_setattr.attr = PDS_VDPA_ATTR_MAC,
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ ether_addr_copy(cmd.vdpa_setattr.mac, mac);
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa_setattr),
+ , 0);
+ if (err)
+ dev_dbg(dev, "Failed to set mac address %pM, status %d: %pe\n",
+ mac, comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_set_max_vq_pairs(struct pds_vdpa_device *pdsv, u16 max_vqp)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR,
+ .vdpa_setattr.vdpa_index = pdsv->vdpa_index,
+ .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
[PATCH v7 virtio 10/11] pds_vdpa: subscribe to the pds_core events
Register for the pds_core's notification events, primarily to
find out when the FW has been reset so we can pass this on
back up the chain.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/pds/vdpa_dev.c | 59 -
drivers/vdpa/pds/vdpa_dev.h | 1 +
2 files changed, 59 insertions(+), 1 deletion(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index 5690ee676ad9..5071a4d58f8d 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -23,6 +23,52 @@ static struct pds_vdpa_device *vdpa_to_pdsv(struct
vdpa_device *vdpa_dev)
return container_of(vdpa_dev, struct pds_vdpa_device, vdpa_dev);
}
+static int pds_vdpa_notify_handler(struct notifier_block *nb,
+ unsigned long ecode,
+ void *data)
+{
+ struct pds_vdpa_device *pdsv = container_of(nb, struct pds_vdpa_device,
nb);
+ struct device *dev = >vdpa_aux->padev->aux_dev.dev;
+
+ dev_dbg(dev, "%s: event code %lu\n", __func__, ecode);
+
+ if (ecode == PDS_EVENT_RESET || ecode == PDS_EVENT_LINK_CHANGE) {
+ if (pdsv->config_cb.callback)
+ pdsv->config_cb.callback(pdsv->config_cb.private);
+ }
+
+ return 0;
+}
+
+static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv)
+{
+ struct device *dev = >vdpa_aux->padev->aux_dev.dev;
+ struct notifier_block *nb = >nb;
+ int err;
+
+ if (!nb->notifier_call) {
+ nb->notifier_call = pds_vdpa_notify_handler;
+ err = pdsc_register_notify(nb);
+ if (err) {
+ nb->notifier_call = NULL;
+ dev_err(dev, "failed to register pds event handler:
%ps\n",
+ ERR_PTR(err));
+ return -EINVAL;
+ }
+ dev_dbg(dev, "pds event handler registered\n");
+ }
+
+ return 0;
+}
+
+static void pds_vdpa_unregister_event_handler(struct pds_vdpa_device *pdsv)
+{
+ if (pdsv->nb.notifier_call) {
+ pdsc_unregister_notify(>nb);
+ pdsv->nb.notifier_call = NULL;
+ }
+}
+
static int pds_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
u64 desc_addr, u64 driver_addr, u64
device_addr)
{
@@ -595,6 +641,12 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
pdsv->vdpa_dev.mdev = _aux->vdpa_mdev;
+ err = pds_vdpa_register_event_handler(pdsv);
+ if (err) {
+ dev_err(dev, "Failed to register for PDS events: %pe\n",
ERR_PTR(err));
+ goto err_unmap;
+ }
+
/* We use the _vdpa_register_device() call rather than the
* vdpa_register_device() to avoid a deadlock because our
* dev_add() is called with the vdpa_dev_lock already set
@@ -603,13 +655,15 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
err = _vdpa_register_device(>vdpa_dev, pdsv->num_vqs);
if (err) {
dev_err(dev, "Failed to register to vDPA bus: %pe\n",
ERR_PTR(err));
- goto err_unmap;
+ goto err_unevent;
}
pds_vdpa_debugfs_add_vdpadev(vdpa_aux);
return 0;
+err_unevent:
+ pds_vdpa_unregister_event_handler(pdsv);
err_unmap:
put_device(>vdpa_dev.dev);
vdpa_aux->pdsv = NULL;
@@ -619,8 +673,11 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
struct vdpa_device *vdpa_dev)
{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
struct pds_vdpa_aux *vdpa_aux;
+ pds_vdpa_unregister_event_handler(pdsv);
+
vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
_vdpa_unregister_device(vdpa_dev);
diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h
index 25c1d192f0ef..a1bc37de9537 100644
--- a/drivers/vdpa/pds/vdpa_dev.h
+++ b/drivers/vdpa/pds/vdpa_dev.h
@@ -40,6 +40,7 @@ struct pds_vdpa_device {
u8 vdpa_index; /* rsvd for future subdevice use */
u8 num_vqs; /* num vqs in use */
struct vdpa_callback config_cb;
+ struct notifier_block nb;
};
#define PDS_VDPA_PACKED_INVERT_IDX 0x8000
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v7 virtio 07/11] pds_vdpa: virtio bar setup for vdpa
Prep and use the "modern" virtio bar utilities to get our
virtio config space ready.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/pds/aux_drv.c | 25 +
drivers/vdpa/pds/aux_drv.h | 3 +++
2 files changed, 28 insertions(+)
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
index aa748cf55d2b..0c4a135b1484 100644
--- a/drivers/vdpa/pds/aux_drv.c
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -4,6 +4,7 @@
#include
#include
#include
+#include
#include
#include
@@ -19,12 +20,22 @@ static const struct auxiliary_device_id pds_vdpa_id_table[]
= {
{},
};
+static int pds_vdpa_device_id_check(struct pci_dev *pdev)
+{
+ if (pdev->device != PCI_DEVICE_ID_PENSANDO_VDPA_VF ||
+ pdev->vendor != PCI_VENDOR_ID_PENSANDO)
+ return -ENODEV;
+
+ return PCI_DEVICE_ID_PENSANDO_VDPA_VF;
+}
+
static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
const struct auxiliary_device_id *id)
{
struct pds_auxiliary_dev *padev =
container_of(aux_dev, struct pds_auxiliary_dev, aux_dev);
+ struct device *dev = _dev->dev;
struct pds_vdpa_aux *vdpa_aux;
int err;
@@ -41,8 +52,21 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
if (err)
goto err_free_mem;
+ /* Find the virtio configuration */
+ vdpa_aux->vd_mdev.pci_dev = padev->vf_pdev;
+ vdpa_aux->vd_mdev.device_id_check = pds_vdpa_device_id_check;
+ vdpa_aux->vd_mdev.dma_mask = DMA_BIT_MASK(PDS_CORE_ADDR_LEN);
+ err = vp_modern_probe(_aux->vd_mdev);
+ if (err) {
+ dev_err(dev, "Unable to probe for virtio configuration: %pe\n",
+ ERR_PTR(err));
+ goto err_free_mgmt_info;
+ }
+
return 0;
+err_free_mgmt_info:
+ pci_free_irq_vectors(padev->vf_pdev);
err_free_mem:
kfree(vdpa_aux);
auxiliary_set_drvdata(aux_dev, NULL);
@@ -55,6 +79,7 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
struct device *dev = _dev->dev;
+ vp_modern_remove(_aux->vd_mdev);
pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
kfree(vdpa_aux);
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
index dcec782e79eb..99e0ff340bfa 100644
--- a/drivers/vdpa/pds/aux_drv.h
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -4,6 +4,8 @@
#ifndef _AUX_DRV_H_
#define _AUX_DRV_H_
+#include
+
#define PDS_VDPA_DRV_DESCRIPTION"AMD/Pensando vDPA VF Device Driver"
#define PDS_VDPA_DRV_NAME KBUILD_MODNAME
@@ -16,6 +18,7 @@ struct pds_vdpa_aux {
int vf_id;
struct dentry *dentry;
+ struct virtio_pci_modern_device vd_mdev;
int nintrs;
};
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v7 virtio 09/11] pds_vdpa: add support for vdpa and vdpamgmt interfaces
This is the vDPA device support, where we advertise that we can
support the virtio queues and deal with the configuration work
through the pds_core's adminq.
Signed-off-by: Shannon Nelson
---
Note: this had previously been Acked-by Jason Wang, but changed enough
in v6 that I felt it needs a new Ack.
drivers/vdpa/pds/aux_drv.c | 15 +
drivers/vdpa/pds/aux_drv.h | 1 +
drivers/vdpa/pds/debugfs.c | 263
drivers/vdpa/pds/debugfs.h | 5 +
drivers/vdpa/pds/vdpa_dev.c | 606 +++-
drivers/vdpa/pds/vdpa_dev.h | 4 +-
6 files changed, 892 insertions(+), 2 deletions(-)
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
index 0c4a135b1484..186e9ee22eb1 100644
--- a/drivers/vdpa/pds/aux_drv.c
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -63,8 +63,21 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
goto err_free_mgmt_info;
}
+ /* Let vdpa know that we can provide devices */
+ err = vdpa_mgmtdev_register(_aux->vdpa_mdev);
+ if (err) {
+ dev_err(dev, "%s: Failed to initialize vdpa_mgmt interface:
%pe\n",
+ __func__, ERR_PTR(err));
+ goto err_free_virtio;
+ }
+
+ pds_vdpa_debugfs_add_pcidev(vdpa_aux);
+ pds_vdpa_debugfs_add_ident(vdpa_aux);
+
return 0;
+err_free_virtio:
+ vp_modern_remove(_aux->vd_mdev);
err_free_mgmt_info:
pci_free_irq_vectors(padev->vf_pdev);
err_free_mem:
@@ -79,9 +92,11 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
struct device *dev = _dev->dev;
+ vdpa_mgmtdev_unregister(_aux->vdpa_mdev);
vp_modern_remove(_aux->vd_mdev);
pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
+ pds_vdpa_debugfs_del_vdpadev(vdpa_aux);
kfree(vdpa_aux);
auxiliary_set_drvdata(aux_dev, NULL);
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
index 99e0ff340bfa..26b75344156e 100644
--- a/drivers/vdpa/pds/aux_drv.h
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -13,6 +13,7 @@ struct pds_vdpa_aux {
struct pds_auxiliary_dev *padev;
struct vdpa_mgmt_dev vdpa_mdev;
+ struct pds_vdpa_device *pdsv;
struct pds_vdpa_ident ident;
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
index d91dceb07380..21a0dc0cb607 100644
--- a/drivers/vdpa/pds/debugfs.c
+++ b/drivers/vdpa/pds/debugfs.c
@@ -10,6 +10,7 @@
#include
#include "aux_drv.h"
+#include "vdpa_dev.h"
#include "debugfs.h"
static struct dentry *dbfs_dir;
@@ -24,3 +25,265 @@ void pds_vdpa_debugfs_destroy(void)
debugfs_remove_recursive(dbfs_dir);
dbfs_dir = NULL;
}
+
+#define PRINT_SBIT_NAME(__seq, __f, __name) \
+ do {\
+ if ((__f) & (__name)) \
+ seq_printf(__seq, " %s",
[PATCH v7 virtio 05/11] pds_vdpa: new adminq entries
Add new adminq definitions in support for vDPA operations.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
include/linux/pds/pds_adminq.h | 226 +
1 file changed, 226 insertions(+)
diff --git a/include/linux/pds/pds_adminq.h b/include/linux/pds/pds_adminq.h
index 61b0a8634e1a..bcba7fda3cc9 100644
--- a/include/linux/pds/pds_adminq.h
+++ b/include/linux/pds/pds_adminq.h
@@ -605,6 +605,219 @@ struct pds_core_q_init_comp {
u8 color;
};
+/*
+ * enum pds_vdpa_cmd_opcode - vDPA Device commands
+ */
+enum pds_vdpa_cmd_opcode {
+ PDS_VDPA_CMD_INIT = 48,
+ PDS_VDPA_CMD_IDENT = 49,
+ PDS_VDPA_CMD_RESET = 51,
+ PDS_VDPA_CMD_VQ_RESET = 52,
+ PDS_VDPA_CMD_VQ_INIT= 53,
+ PDS_VDPA_CMD_STATUS_UPDATE = 54,
+ PDS_VDPA_CMD_SET_FEATURES = 55,
+ PDS_VDPA_CMD_SET_ATTR = 56,
+};
+
+/**
+ * struct pds_vdpa_cmd - generic command
+ * @opcode:Opcode
+ * @vdpa_index:Index for vdpa subdevice
+ * @vf_id: VF id
+ */
+struct pds_vdpa_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+};
+
+/**
+ * struct pds_vdpa_init_cmd - INIT command
+ * @opcode:Opcode PDS_VDPA_CMD_INIT
+ * @vdpa_index: Index for vdpa subdevice
+ * @vf_id: VF id
+ */
+struct pds_vdpa_init_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+};
+
+/**
+ * struct pds_vdpa_ident - vDPA identification data
+ * @hw_features: vDPA features supported by device
+ * @max_vqs: max queues available (2 queues for a single queuepair)
+ * @max_qlen: log(2) of maximum number of descriptors
+ * @min_qlen: log(2) of minimum number of descriptors
+ *
+ * This struct is used in a DMA block that is set up for the PDS_VDPA_CMD_IDENT
+ * transaction. Set up the DMA block and send the address in the IDENT cmd
+ * data, the DSC will write the ident information, then we can remove the DMA
+ * block after reading the answer. If the completion status is 0, then there
+ * is valid information, else there was an error and the data should be
invalid.
+ */
+struct pds_vdpa_ident {
+ __le64 hw_features;
+ __le16 max_vqs;
+ __le16 max_qlen;
+ __le16 min_qlen;
+};
+
+/**
+ * struct pds_vdpa_ident_cmd - IDENT command
+ * @opcode:Opcode PDS_VDPA_CMD_IDENT
+ * @rsvd: Word boundary padding
+ * @vf_id: VF id
+ * @len: length of ident info DMA space
+ * @ident_pa: address for DMA of ident info (struct pds_vdpa_ident)
+ * only used for this transaction, then forgotten by DSC
+ */
+struct pds_vdpa_ident_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 vf_id;
+ __le32 len;
+ __le64 ident_pa;
+};
+
+/**
+ * struct pds_vdpa_status_cmd - STATUS_UPDATE command
+ * @opcode:Opcode PDS_VDPA_CMD_STATUS_UPDATE
+ * @vdpa_index: Index for vdpa subdevice
+ * @vf_id: VF id
+ * @status:new status bits
+ */
+struct pds_vdpa_status_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+ u8 status;
+};
+
+/**
+ * enum pds_vdpa_attr - List of VDPA device attributes
+ * @PDS_VDPA_ATTR_MAC: MAC address
+ * @PDS_VDPA_ATTR_MAX_VQ_PAIRS: Max virtqueue pairs
+ */
+enum pds_vdpa_attr {
+ PDS_VDPA_ATTR_MAC = 1,
+ PDS_VDPA_ATTR_MAX_VQ_PAIRS = 2,
+};
+
+/**
+ * struct pds_vdpa_setattr_cmd - SET_ATTR command
+ * @opcode:Opcode PDS_VDPA_CMD_SET_ATTR
+ * @vdpa_index:Index for vdpa subdevice
+ * @vf_id: VF id
+ * @attr: attribute to be changed (enum pds_vdpa_attr)
+ * @pad: Word boundary padding
+ * @mac: new mac address to be assigned as vdpa device address
+ * @max_vq_pairs: new limit of virtqueue pairs
+ */
+struct pds_vdpa_setattr_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+ u8 attr;
+ u8 pad[3];
+ union {
+ u8 mac[6];
+ __le16 max_vq_pairs;
+ } __packed;
+};
+
+/**
+ * struct pds_vdpa_vq_init_cmd - queue init command
+ * @opcode: Opcode PDS_VDPA_CMD_VQ_INIT
+ * @vdpa_index:Index for vdpa subdevice
+ * @vf_id: VF id
+ * @qid: Queue id (bit0 clear = rx, bit0 set = tx, qid=N is ctrlq)
+ * @len: log(2) of max descriptor count
+ * @desc_addr: DMA address of descriptor area
+ * @avail_addr:DMA address of available descriptors (aka driver area)
+ * @used_addr: DMA address of used descriptors (aka device area)
+ * @intr_index:interrupt index
+ * @avail_index: initial device position in available ring
+ * @used_index:initial device position in used ring
+ */
+struct pds_vdpa_vq_init_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+ __le16 qid;
+ __le16 len;
+ __le64 desc_addr;
+ __le64
[PATCH v7 virtio 06/11] pds_vdpa: get vdpa management info
Find the vDPA management information from the DSC in order to
advertise it to the vdpa subsystem.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/pds/Makefile | 3 +-
drivers/vdpa/pds/aux_drv.c | 17 ++
drivers/vdpa/pds/aux_drv.h | 7 +++
drivers/vdpa/pds/debugfs.c | 1 +
drivers/vdpa/pds/vdpa_dev.c | 108
drivers/vdpa/pds/vdpa_dev.h | 15 +
6 files changed, 150 insertions(+), 1 deletion(-)
create mode 100644 drivers/vdpa/pds/vdpa_dev.c
create mode 100644 drivers/vdpa/pds/vdpa_dev.h
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
index a9cd2f450ae1..13b50394ec64 100644
--- a/drivers/vdpa/pds/Makefile
+++ b/drivers/vdpa/pds/Makefile
@@ -3,6 +3,7 @@
obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
-pds_vdpa-y := aux_drv.o
+pds_vdpa-y := aux_drv.o \
+ vdpa_dev.o
pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
index e4a0ad61ea22..aa748cf55d2b 100644
--- a/drivers/vdpa/pds/aux_drv.c
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -3,6 +3,7 @@
#include
#include
+#include
#include
#include
@@ -11,6 +12,7 @@
#include "aux_drv.h"
#include "debugfs.h"
+#include "vdpa_dev.h"
static const struct auxiliary_device_id pds_vdpa_id_table[] = {
{ .name = PDS_VDPA_DEV_NAME, },
@@ -24,15 +26,28 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
struct pds_auxiliary_dev *padev =
container_of(aux_dev, struct pds_auxiliary_dev, aux_dev);
struct pds_vdpa_aux *vdpa_aux;
+ int err;
vdpa_aux = kzalloc(sizeof(*vdpa_aux), GFP_KERNEL);
if (!vdpa_aux)
return -ENOMEM;
vdpa_aux->padev = padev;
+ vdpa_aux->vf_id = pci_iov_vf_id(padev->vf_pdev);
auxiliary_set_drvdata(aux_dev, vdpa_aux);
+ /* Get device ident info and set up the vdpa_mgmt_dev */
+ err = pds_vdpa_get_mgmt_info(vdpa_aux);
+ if (err)
+ goto err_free_mem;
+
return 0;
+
+err_free_mem:
+ kfree(vdpa_aux);
+ auxiliary_set_drvdata(aux_dev, NULL);
+
+ return err;
}
static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
@@ -40,6 +55,8 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
struct device *dev = _dev->dev;
+ pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
+
kfree(vdpa_aux);
auxiliary_set_drvdata(aux_dev, NULL);
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
index f1e99359424e..dcec782e79eb 100644
--- a/drivers/vdpa/pds/aux_drv.h
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -10,6 +10,13 @@
struct pds_vdpa_aux {
struct pds_auxiliary_dev *padev;
+ struct vdpa_mgmt_dev vdpa_mdev;
+
+ struct pds_vdpa_ident ident;
+
+ int vf_id;
struct dentry *dentry;
+
+ int nintrs;
};
#endif /* _AUX_DRV_H_ */
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
index 5be22fb7a76a..d91dceb07380 100644
--- a/drivers/vdpa/pds/debugfs.c
+++ b/drivers/vdpa/pds/debugfs.c
@@ -2,6 +2,7 @@
/* Copyright(c) 2023 Advanced Micro Devices, Inc */
#include
+#include
#include
#include
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
new file mode 100644
index ..0f0f0ab8b811
--- /dev/null
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include "vdpa_dev.h"
+#include "aux_drv.h"
+
+static struct virtio_device_id pds_vdpa_id_table[] = {
+ {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
+ {0},
+};
+
+static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
+ const struct vdpa_dev_set_config *add_config)
+{
+ return -EOPNOTSUPP;
+}
+
+static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
+struct vdpa_device *vdpa_dev)
+{
+}
+
+static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = {
+ .dev_add = pds_vdpa_dev_add,
+ .dev_del = pds_vdpa_dev_del
+};
+
+int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
+{
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_ident.opcode = PDS_VDPA_CMD_IDENT,
+ .vdpa_ident.vf_id = cpu_to_le16(vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ struct vdpa_mgmt_dev *mgmt;
+ struct pci_dev *pf_pdev;
+ struct device *pf_dev;
+ struct pci_dev *pdev;
+ dma_addr_t ident_pa;
+ struct device *dev;
+ u16 dev_intrs;
+ u16 max_vqs;
+ int err;
+
+ dev = _aux->padev->aux_dev.dev;
+ pdev = vdpa_aux->padev->vf_pdev;
+ mgmt = _aux->vdpa_mdev;
+
+ /* Get resource info through the PF's adminq. It
[PATCH v7 virtio 03/11] pds_vdpa: Add new vDPA driver for AMD/Pensando DSC
This is the initial auxiliary driver framework for a new vDPA
device driver, an auxiliary_bus client of the pds_core driver.
The pds_core driver supplies the PCI services for the VF device
and for accessing the adminq in the PF device.
This patch adds the very basics of registering for the auxiliary
device and setting up debugfs entries.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/Makefile | 1 +
drivers/vdpa/pds/Makefile | 8
drivers/vdpa/pds/aux_drv.c | 83 ++
drivers/vdpa/pds/aux_drv.h | 15 ++
drivers/vdpa/pds/debugfs.c | 25 ++
drivers/vdpa/pds/debugfs.h | 12 +
include/linux/pds/pds_common.h | 2 +
7 files changed, 146 insertions(+)
create mode 100644 drivers/vdpa/pds/Makefile
create mode 100644 drivers/vdpa/pds/aux_drv.c
create mode 100644 drivers/vdpa/pds/aux_drv.h
create mode 100644 drivers/vdpa/pds/debugfs.c
create mode 100644 drivers/vdpa/pds/debugfs.h
diff --git a/drivers/vdpa/Makefile b/drivers/vdpa/Makefile
index 59396ff2a318..8f53c6f3cca7 100644
--- a/drivers/vdpa/Makefile
+++ b/drivers/vdpa/Makefile
@@ -7,3 +7,4 @@ obj-$(CONFIG_MLX5_VDPA) += mlx5/
obj-$(CONFIG_VP_VDPA)+= virtio_pci/
obj-$(CONFIG_ALIBABA_ENI_VDPA) += alibaba/
obj-$(CONFIG_SNET_VDPA) += solidrun/
+obj-$(CONFIG_PDS_VDPA) += pds/
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
new file mode 100644
index ..a9cd2f450ae1
--- /dev/null
+++ b/drivers/vdpa/pds/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# Copyright(c) 2023 Advanced Micro Devices, Inc
+
+obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
+
+pds_vdpa-y := aux_drv.o
+
+pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
new file mode 100644
index ..e4a0ad61ea22
--- /dev/null
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include "aux_drv.h"
+#include "debugfs.h"
+
+static const struct auxiliary_device_id pds_vdpa_id_table[] = {
+ { .name = PDS_VDPA_DEV_NAME, },
+ {},
+};
+
+static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
+ const struct auxiliary_device_id *id)
+
+{
+ struct pds_auxiliary_dev *padev =
+ container_of(aux_dev, struct pds_auxiliary_dev, aux_dev);
+ struct pds_vdpa_aux *vdpa_aux;
+
+ vdpa_aux = kzalloc(sizeof(*vdpa_aux), GFP_KERNEL);
+ if (!vdpa_aux)
+ return -ENOMEM;
+
+ vdpa_aux->padev = padev;
+ auxiliary_set_drvdata(aux_dev, vdpa_aux);
+
+ return 0;
+}
+
+static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
+{
+ struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
+ struct device *dev = _dev->dev;
+
+ kfree(vdpa_aux);
+ auxiliary_set_drvdata(aux_dev, NULL);
+
+ dev_info(dev, "Removed\n");
+}
+
+static struct auxiliary_driver pds_vdpa_driver = {
+ .name = PDS_DEV_TYPE_VDPA_STR,
+ .probe = pds_vdpa_probe,
+ .remove = pds_vdpa_remove,
+ .id_table = pds_vdpa_id_table,
+};
+
+static void __exit pds_vdpa_cleanup(void)
+{
+ auxiliary_driver_unregister(_vdpa_driver);
+
+ pds_vdpa_debugfs_destroy();
+}
+module_exit(pds_vdpa_cleanup);
+
+static int __init pds_vdpa_init(void)
+{
+ int err;
+
+ pds_vdpa_debugfs_create();
+
+ err = auxiliary_driver_register(_vdpa_driver);
+ if (err) {
+ pr_err("%s: aux driver register failed: %pe\n",
+ PDS_VDPA_DRV_NAME, ERR_PTR(err));
+ pds_vdpa_debugfs_destroy();
+ }
+
+ return err;
+}
+module_init(pds_vdpa_init);
+
+MODULE_DESCRIPTION(PDS_VDPA_DRV_DESCRIPTION);
+MODULE_AUTHOR("Advanced Micro Devices, Inc");
+MODULE_LICENSE("GPL");
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
new file mode 100644
index ..f1e99359424e
--- /dev/null
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#ifndef _AUX_DRV_H_
+#define _AUX_DRV_H_
+
+#define PDS_VDPA_DRV_DESCRIPTION"AMD/Pensando vDPA VF Device Driver"
+#define PDS_VDPA_DRV_NAME KBUILD_MODNAME
+
+struct pds_vdpa_aux {
+ struct pds_auxiliary_dev *padev;
+
+ struct dentry *dentry;
+};
+#endif /* _AUX_DRV_H_ */
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
new file mode 100644
index ..5be22fb7a76a
--- /dev/null
+++ b/drivers/vdpa/pds/debugfs.c
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+
+#include
+#include
+#include
+#include
+
+#include "aux_drv.h"
+#include "debugfs.h"
+
+static struct dentry *dbfs_dir;
+
+void
[PATCH v7 virtio 04/11] pds_vdpa: move enum from common to adminq header
The pds_core_logical_qtype enum and IFNAMSIZ are not needed
in the common PDS header, only needed when working with the
adminq, so move them to the adminq header.
Note: This patch might conflict with pds_vfio patches that are
in review, depending on which patchset gets pulled first.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
include/linux/pds/pds_adminq.h | 21 +
include/linux/pds/pds_common.h | 21 -
2 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/include/linux/pds/pds_adminq.h b/include/linux/pds/pds_adminq.h
index 98a60ce87b92..61b0a8634e1a 100644
--- a/include/linux/pds/pds_adminq.h
+++ b/include/linux/pds/pds_adminq.h
@@ -222,6 +222,27 @@ enum pds_core_lif_type {
PDS_CORE_LIF_TYPE_DEFAULT = 0,
};
+#define PDS_CORE_IFNAMSIZ 16
+
+/**
+ * enum pds_core_logical_qtype - Logical Queue Types
+ * @PDS_CORE_QTYPE_ADMINQ:Administrative Queue
+ * @PDS_CORE_QTYPE_NOTIFYQ: Notify Queue
+ * @PDS_CORE_QTYPE_RXQ: Receive Queue
+ * @PDS_CORE_QTYPE_TXQ: Transmit Queue
+ * @PDS_CORE_QTYPE_EQ:Event Queue
+ * @PDS_CORE_QTYPE_MAX: Max queue type supported
+ */
+enum pds_core_logical_qtype {
+ PDS_CORE_QTYPE_ADMINQ = 0,
+ PDS_CORE_QTYPE_NOTIFYQ = 1,
+ PDS_CORE_QTYPE_RXQ = 2,
+ PDS_CORE_QTYPE_TXQ = 3,
+ PDS_CORE_QTYPE_EQ = 4,
+
+ PDS_CORE_QTYPE_MAX = 16 /* don't change - used in struct size */
+};
+
/**
* union pds_core_lif_config - LIF configuration
* @state: LIF state (enum pds_core_lif_state)
diff --git a/include/linux/pds/pds_common.h b/include/linux/pds/pds_common.h
index 2a0d1669cfd0..435c8e8161c2 100644
--- a/include/linux/pds/pds_common.h
+++ b/include/linux/pds/pds_common.h
@@ -41,27 +41,6 @@ enum pds_core_vif_types {
#define PDS_VDPA_DEV_NAME PDS_CORE_DRV_NAME "." PDS_DEV_TYPE_VDPA_STR
-#define PDS_CORE_IFNAMSIZ 16
-
-/**
- * enum pds_core_logical_qtype - Logical Queue Types
- * @PDS_CORE_QTYPE_ADMINQ:Administrative Queue
- * @PDS_CORE_QTYPE_NOTIFYQ: Notify Queue
- * @PDS_CORE_QTYPE_RXQ: Receive Queue
- * @PDS_CORE_QTYPE_TXQ: Transmit Queue
- * @PDS_CORE_QTYPE_EQ:Event Queue
- * @PDS_CORE_QTYPE_MAX: Max queue type supported
- */
-enum pds_core_logical_qtype {
- PDS_CORE_QTYPE_ADMINQ = 0,
- PDS_CORE_QTYPE_NOTIFYQ = 1,
- PDS_CORE_QTYPE_RXQ = 2,
- PDS_CORE_QTYPE_TXQ = 3,
- PDS_CORE_QTYPE_EQ = 4,
-
- PDS_CORE_QTYPE_MAX = 16 /* don't change - used in struct size */
-};
-
int pdsc_register_notify(struct notifier_block *nb);
void pdsc_unregister_notify(struct notifier_block *nb);
void *pdsc_get_pf_struct(struct pci_dev *vf_pdev);
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v7 virtio 01/11] virtio: allow caller to override device id in vp_modern
To add a bit of vendor flexibility with various virtio based devices,
allow the caller to check for a different device id. This adds a function
pointer field to struct virtio_pci_modern_device to specify an override
device id check. If defined by the driver, this function will be called
to check that the PCI device is the vendor's expected device, and will
return the found device id to be stored in mdev->id.device. This allows
vendors with alternative vendor device ids to use this library on their
own device BAR.
Note: A lot of the diff in this is simply indenting the existing code
into an else block.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/virtio/virtio_pci_modern_dev.c | 30 --
include/linux/virtio_pci_modern.h | 3 +++
2 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/drivers/virtio/virtio_pci_modern_dev.c
b/drivers/virtio/virtio_pci_modern_dev.c
index 869cb46bef96..9b2d6614de67 100644
--- a/drivers/virtio/virtio_pci_modern_dev.c
+++ b/drivers/virtio/virtio_pci_modern_dev.c
@@ -218,21 +218,29 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
int err, common, isr, notify, device;
u32 notify_length;
u32 notify_offset;
+ int devid;
check_offsets();
- /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */
- if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)
- return -ENODEV;
-
- if (pci_dev->device < 0x1040) {
- /* Transitional devices: use the PCI subsystem device id as
-* virtio device id, same as legacy driver always did.
-*/
- mdev->id.device = pci_dev->subsystem_device;
+ if (mdev->device_id_check) {
+ devid = mdev->device_id_check(pci_dev);
+ if (devid < 0)
+ return devid;
+ mdev->id.device = devid;
} else {
- /* Modern devices: simply use PCI device id, but start from
0x1040. */
- mdev->id.device = pci_dev->device - 0x1040;
+ /* We only own devices >= 0x1000 and <= 0x107f: leave the rest.
*/
+ if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)
+ return -ENODEV;
+
+ if (pci_dev->device < 0x1040) {
+ /* Transitional devices: use the PCI subsystem device
id as
+* virtio device id, same as legacy driver always did.
+*/
+ mdev->id.device = pci_dev->subsystem_device;
+ } else {
+ /* Modern devices: simply use PCI device id, but start
from 0x1040. */
+ mdev->id.device = pci_dev->device - 0x1040;
+ }
}
mdev->id.vendor = pci_dev->subsystem_vendor;
diff --git a/include/linux/virtio_pci_modern.h
b/include/linux/virtio_pci_modern.h
index c4eeb79b0139..e7b1db1dd0bb 100644
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
@@ -38,6 +38,9 @@ struct virtio_pci_modern_device {
int modern_bars;
struct virtio_device_id id;
+
+ /* optional check for vendor virtio device, returns dev_id or -ERRNO */
+ int (*device_id_check)(struct pci_dev *pdev);
};
/*
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v7 virtio 00/11] pds_vdpa driver
This patchset implements a new module for the AMD/Pensando DSC that supports vDPA services on PDS Core VF devices. This code is based on and depends on include files from the pds_core driver described here[0]. The pds_core driver creates the auxiliary_bus devices that this module connects to, and this creates vdpa devices for use by the vdpa module. The first version of this driver was a part of the original pds_core RFC [1] but has since been reworked to pull out the PCI driver and to make better use of the virtio and virtio_net configuration spaces made available by the DSC's PCI configuration. As the device development has progressed, the ability to rely on the virtio config spaces has grown. This patchset includes a modification to the existing vp_modern_probe() which implements overrides for the PCI device id check and the DMA mask. These are intended to be used with vendor vDPA devices that implement enough of the virtio config space to be used directly, but don't use the virtio device id. To use this module, enable the VFs and turn on the vDPA services in the pds_core PF, then use the 'vdpa' utility to create devices for use by virtio_vdpa or vhost_vdpa: echo 1 > /sys/bus/pci/drivers/pds_core/$PF_BDF/sriov_numvfs devlink dev param set pci/$PF_BDF name enable_vnet value true cmode runtime PDS_VDPA_MGMT=`vdpa mgmtdev show | grep vDPA | head -1 | cut -d: -f1` vdpa dev add name vdpa1 mgmtdev $PDS_VDPA_MGMT mac 00:11:22:33:44:55 [0] Link: https://lore.kernel.org/netdev/20230419170427.1108-1-shannon.nel...@amd.com/ [1] Link: https://lore.kernel.org/netdev/20221118225656.48309-1-snel...@pensando.io/ Changes: v7: - fixed undefined err value noticed by Simon and the kernel test robot v6: Link: https://lore.kernel.org/netdev/20230516025521.43352-1-shannon.nel...@amd.com/ - removed misleading comments from pds_vdpa_notify_handler() - added Kconfig "select VIRTIO_PCI_LIB" and "depends on PCI_MSI" - changed pds_vdpa_set_status() to use an adminq cmd to bypass a FW sequencing issue - added map vq notify after features_ok for correct placement after feature negotiation - added support for dev_add() option VDPA_ATTR_DEV_FEATURES - reworked get/set_vq_state() to better support packed, split, and legacy use - dropped Jason's Acked-by on 09/11 because of these last four changes v5: Link: https://lore.kernel.org/virtualization/20230503181240.14009-1-shannon.nel...@amd.com/ - split dma_mask and device_id_check() into separate patches - simplify use of dma_mask into a single line change - changed test of VIRTIO_F_RING_PACKED to use BIT_ULL() v4: Link: https://lore.kernel.org/virtualization/20230425212602.1157-1-shannon.nel...@amd.com/ - rename device_id_check_override() to device_id_check() - make device_id_check() return the device_id found and checked - removed pds_vdpa.h, put its adminq changes into pds_adminq.h - added a patch to separate out the adminq changes - added a patch to move an adminq enum from pds_common.h to pds_adminq.h - moved adminq calls for get/set_vq_state into cmds.c - limit max_vqs by number of msix available - don't increment nintrs for CVQ, it should already be covered from max_vqs - pds_core API related rework following pds_core inclusion to net-next - use non-debugfs method to find PF pci address in pds_vdpa.rst instructions v3: Link: https://lore.kernel.org/netdev/20230330192313.62018-1-shannon.nel...@amd.com/ - added a patch to modify vp_modern_probe() such that specific device id and DMA mask overrides can be used - add pds_vdpa.rst into index file - dev_dbg instead of dev_err on most of the adminq commands - rework use of pds_vdpa_cmd_reset() and pds_vdpa_init_hw() for better firmware setup in start-stop-start scenarios - removed unused pds_vdpa_cmd_set_features(), we can rely on vp_modern_set_features() - remove unused hw_qtype and hw_qindex from pds_vdpa_vq_info - reworked debugfs print_feature_bits to also print unknown bits - changed use of PAGE_SIZE to local PDS_PAGE_SIZE to keep with FW layout needs without regard to kernel PAGE_SIZE configuration v2: https://lore.kernel.org/netdev/20230309013046.23523-1-shannon.nel...@amd.com/ - removed PCI driver code - replaced home-grown event listener with notifier - replaced many adminq uses with direct virtio_net config access - reworked irqs to follow virtio layout - removed local_mac_bit logic - replaced uses of devm_ interfaces as suggested in pds_core reviews - updated copyright strings to reflect the new owner Shannon Nelson (11): virtio: allow caller to override device id in vp_modern virtio: allow caller to override device DMA mask in vp_modern pds_vdpa: Add new vDPA driver for AMD/Pensando DSC pds_vdpa: move enum from common to adminq header pds_vdpa: new adminq entries pds_vdpa: get vdpa management info pds_vdpa: virtio bar setup for vdpa pds_vdpa: add vdpa config client commands pds_vdpa: add support for vdpa and vdpamgmt
[PATCH v7 virtio 02/11] virtio: allow caller to override device DMA mask in vp_modern
To add a bit of vendor flexibility with various virtio based devices,
allow the caller to specify a different DMA mask. This adds a dma_mask
field to struct virtio_pci_modern_device. If defined by the driver,
this mask will be used in a call to dma_set_mask_and_coherent() instead
of the traditional DMA_BIT_MASK(64). This allows limiting the DMA space
on vendor devices with address limitations.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/virtio/virtio_pci_modern_dev.c | 3 ++-
include/linux/virtio_pci_modern.h | 3 +++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/drivers/virtio/virtio_pci_modern_dev.c
b/drivers/virtio/virtio_pci_modern_dev.c
index 9b2d6614de67..aad7d9296e77 100644
--- a/drivers/virtio/virtio_pci_modern_dev.c
+++ b/drivers/virtio/virtio_pci_modern_dev.c
@@ -268,7 +268,8 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
return -EINVAL;
}
- err = dma_set_mask_and_coherent(_dev->dev, DMA_BIT_MASK(64));
+ err = dma_set_mask_and_coherent(_dev->dev,
+ mdev->dma_mask ? : DMA_BIT_MASK(64));
if (err)
err = dma_set_mask_and_coherent(_dev->dev,
DMA_BIT_MASK(32));
diff --git a/include/linux/virtio_pci_modern.h
b/include/linux/virtio_pci_modern.h
index e7b1db1dd0bb..067ac1d789bc 100644
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
@@ -41,6 +41,9 @@ struct virtio_pci_modern_device {
/* optional check for vendor virtio device, returns dev_id or -ERRNO */
int (*device_id_check)(struct pci_dev *pdev);
+
+ /* optional mask for devices with limited DMA space */
+ u64 dma_mask;
};
/*
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH v6 virtio 11/11] pds_vdpa: pds_vdps.rst and Kconfig
On 5/16/23 7:24 PM, kernel test robot wrote:
Hi Shannon,
kernel test robot noticed the following build warnings:
[auto build test WARNING on linus/master]
[also build test WARNING on v6.4-rc2 next-20230516]
[cannot apply to mst-vhost/linux-next]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url:
https://github.com/intel-lab-lkp/linux/commits/Shannon-Nelson/virtio-allow-caller-to-override-device-id-in-vp_modern/20230516-110049
base: linus/master
patch link:
https://lore.kernel.org/r/20230516025521.43352-12-shannon.nelson%40amd.com
patch subject: [PATCH v6 virtio 11/11] pds_vdpa: pds_vdps.rst and Kconfig
config: arm64-allmodconfig
compiler: clang version 17.0.0 (https://github.com/llvm/llvm-project
b0fb98227c90adf2536c9ad644a74d5e9296)
reproduce (this is a W=1 build):
wget
https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O
~/bin/make.cross
chmod +x ~/bin/make.cross
# install arm64 cross compiling tool for clang build
# apt-get install binutils-aarch64-linux-gnu
#
https://github.com/intel-lab-lkp/linux/commit/4942556344e0c7cdf4719dfbd0e17c0f2b620b30
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review
Shannon-Nelson/virtio-allow-caller-to-override-device-id-in-vp_modern/20230516-110049
git checkout 4942556344e0c7cdf4719dfbd0e17c0f2b620b30
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1
O=build_dir ARCH=arm64 olddefconfig
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1
O=build_dir ARCH=arm64 SHELL=/bin/bash drivers/acpi/ drivers/infiniband/sw/rxe/
drivers/staging/media/deprecated/atmel/ drivers/vdpa/pds/
If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot
| Link:
https://lore.kernel.org/oe-kbuild-all/202305171042.ajukaxnh-...@intel.com/
Thank you, Mr. Kernel Test Robot, I have this on my list.
sln
All warnings (new ones prefixed by >>):
drivers/vdpa/pds/vdpa_dev.c:568:7: warning: variable 'err' is used
uninitialized whenever 'if' condition is true [-Wsometimes-uninitialized]
if (unsupp_features) {
^~~
drivers/vdpa/pds/vdpa_dev.c:669:9: note: uninitialized use occurs here
return err;
^~~
drivers/vdpa/pds/vdpa_dev.c:568:3: note: remove the 'if' if its condition
is always false
if (unsupp_features) {
^~
drivers/vdpa/pds/vdpa_dev.c:536:9: note: initialize the variable 'err' to
silence this warning
int err;
^
= 0
1 warning generated.
vim +568 drivers/vdpa/pds/vdpa_dev.c
8b05d813374db7 Shannon Nelson 2023-05-15 524
8b05d813374db7 Shannon Nelson 2023-05-15 525 static int
pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
8b05d813374db7 Shannon Nelson 2023-05-15 526 const
struct vdpa_dev_set_config *add_config)
8b05d813374db7 Shannon Nelson 2023-05-15 527 {
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 528 struct pds_vdpa_aux *vdpa_aux;
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 529 struct pds_vdpa_device *pdsv;
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 530 struct vdpa_mgmt_dev *mgmt;
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 531 u16 fw_max_vqs, vq_pairs;
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 532 struct device *dma_dev;
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 533 struct pci_dev *pdev;
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 534 struct device *dev;
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 535 u8 mac[ETH_ALEN];
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 536 int err;
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 537 int i;
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 538
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 539 vdpa_aux = container_of(mdev,
struct pds_vdpa_aux, vdpa_mdev);
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 540 dev =
_aux->padev->aux_dev.dev;
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 541 mgmt = _aux->vdpa_mdev;
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 542
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 543 if (vdpa_aux->pdsv) {
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 544 dev_warn(dev, "Multiple vDPA
devices on a VF is not supported.\n");
8b05d813374db7 Shannon Nelson 2023-05-15 545 return -EOPNOTSUPP;
8b05d813374db7 Shannon Nelson 2023-05-15 546 }
8b05d813374db7 Shannon Nelson 2023-05-15 547
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 548 pdsv = vdpa_alloc_device(struct
pds_vdpa_device, vdpa_dev,
c4cc9ee57a0c5b Shannon Nelson 2023-05-15 549dev,
_vdpa_ops,
Re: [PATCH v2 2/3] vhost: support PACKED when setting-getting vring_base
On 5/16/23 12:49 AM, Stefano Garzarella wrote:
On Mon, May 15, 2023 at 01:41:12PM -0700, Shannon Nelson wrote:
On 5/9/23 1:46 AM, Stefano Garzarella wrote:
On Mon, Apr 24, 2023 at 03:50:30PM -0700, Shannon Nelson via
Virtualization wrote:
Use the right structs for PACKED or split vqs when setting and
getting the vring base.
Signed-off-by: Shannon Nelson
---
drivers/vhost/vhost.c | 18 +-
drivers/vhost/vhost.h | 8 ++--
2 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index f11bdbe4c2c5..f64efda48f21 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -1633,17 +1633,25 @@ long vhost_vring_ioctl(struct vhost_dev
*d, unsigned int ioctl, void __user *arg
r = -EFAULT;
break;
}
- if (s.num > 0x) {
- r = -EINVAL;
- break;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
+ vq->last_avail_idx = s.num & 0x;
+ vq->last_used_idx = (s.num >> 16) & 0x;
+ } else {
+ if (s.num > 0x) {
+ r = -EINVAL;
+ break;
+ }
+ vq->last_avail_idx = s.num;
}
- vq->last_avail_idx = s.num;
/* Forget the cached index value. */
vq->avail_idx = vq->last_avail_idx;
break;
case VHOST_GET_VRING_BASE:
s.index = idx;
- s.num = vq->last_avail_idx;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED))
+ s.num = (u32)vq->last_avail_idx |
((u32)vq->last_used_idx << 16);
+ else
+ s.num = vq->last_avail_idx;
The changes LGTM, but since we are changing the UAPI, should we
update the documentation of VHOST_SET_VRING_BASE and
VHOST_GET_VRING_BASE in include/uapi/linux/vhost.h?
Correct me if I'm wrong, but I don't think we're changing anything in
the UAPI here, just fixing code to work correctly with what is already
happening.
IIUC before this patch VHOST_GET_VRING_BASE and VHOST_SET_VRING_BASE
never worked with packed virtqueue, since we were only handling
last_avail_idx. Now we are supporting packed virtqueue, handling
in vhost_vring_state.num both last_avail_idx and last_used_idx (with
wrap counters).
For example for VHOST_GET_VRING_BASE where is documented that the first
15 bits are last_avail_idx, the 16th the avail_wrap_counter, and the
others are last_used_idx and used_wrap_counter?
Maybe I missed something, but since this is UAPI, IMHO we should
document the parameters of ioctls at least in
include/uapi/linux/vhost.h.
Perhaps Jason already has something written up that could be put in here
from when he first added the wrap_counter a couple of years ago?
sln
Thanks,
Stefano
--
You received this message because you are subscribed to the Google
Groups "Pensando Drivers" group.
To unsubscribe from this group and stop receiving emails from it, send
an email to drivers+unsubscr...@pensando.io.
To view this discussion on the web visit
https://groups.google.com/a/pensando.io/d/msgid/drivers/q6cmfha36sdkgflwwd3pr4sw7rgajag4ahgjbpfjrr76w4o2b6%403yc7zs5u65s4.
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH v6 virtio 09/11] pds_vdpa: add support for vdpa and vdpamgmt interfaces
On 5/16/23 3:35 AM, Simon Horman wrote:
On Mon, May 15, 2023 at 07:55:19PM -0700, Shannon Nelson wrote:
This is the vDPA device support, where we advertise that we can
support the virtio queues and deal with the configuration work
through the pds_core's adminq.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
...
@@ -21,12 +479,156 @@ static struct virtio_device_id pds_vdpa_id_table[] = {
static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
const struct vdpa_dev_set_config *add_config)
{
- return -EOPNOTSUPP;
+ struct pds_vdpa_aux *vdpa_aux;
+ struct pds_vdpa_device *pdsv;
+ struct vdpa_mgmt_dev *mgmt;
+ u16 fw_max_vqs, vq_pairs;
+ struct device *dma_dev;
+ struct pci_dev *pdev;
+ struct device *dev;
+ u8 mac[ETH_ALEN];
+ int err;
+ int i;
+
+ vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
+ dev = _aux->padev->aux_dev.dev;
+ mgmt = _aux->vdpa_mdev;
+
+ if (vdpa_aux->pdsv) {
+ dev_warn(dev, "Multiple vDPA devices on a VF is not
supported.\n");
+ return -EOPNOTSUPP;
+ }
+
+ pdsv = vdpa_alloc_device(struct pds_vdpa_device, vdpa_dev,
+ dev, _vdpa_ops, 1, 1, name, false);
+ if (IS_ERR(pdsv)) {
+ dev_err(dev, "Failed to allocate vDPA structure: %pe\n", pdsv);
+ return PTR_ERR(pdsv);
+ }
+
+ vdpa_aux->pdsv = pdsv;
+ pdsv->vdpa_aux = vdpa_aux;
+
+ pdev = vdpa_aux->padev->vf_pdev;
+ dma_dev = >dev;
+ pdsv->vdpa_dev.dma_dev = dma_dev;
+
+ pdsv->supported_features = mgmt->supported_features;
+
+ if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
+ u64 unsupp_features =
+ add_config->device_features & ~mgmt->supported_features;
+
+ if (unsupp_features) {
+ dev_err(dev, "Unsupported features: %#llx\n",
unsupp_features);
+ goto err_unmap;
Hi Shannon,
clang-16 W=1 reports that
err_unmap will return err
but err is uninitialised here.
Clearly I need to expand my toolset.
Good catch - thanks.
sln
+ }
+
+ pdsv->supported_features = add_config->device_features;
+ }
...
+err_unmap:
+ put_device(>vdpa_dev.dev);
+ vdpa_aux->pdsv = NULL;
+ return err;
}
...
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH v6 virtio 04/11] pds_vdpa: move enum from common to adminq header
On 5/15/23 11:12 PM, Michael S. Tsirkin wrote:
On Mon, May 15, 2023 at 07:55:14PM -0700, Shannon Nelson wrote:
The pds_core_logical_qtype enum and IFNAMSIZ are not needed
in the common PDS header, only needed when working with the
adminq, so move them to the adminq header.
Note: This patch might conflict with pds_vfio patches that are
in review, depending on which patchset gets pulled first.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
It's a bit weird to add code in one patch then move it
in another. Why not start with it in the final location?
Yes, and I usually try to catch those things before they go out :-). In
this case the chunk we're moving was added in the pds_core patchset a
few weeks ago, and since then we have realized it would be better to be
localized to the adminq header. Perhaps this would have been clearer if
this patch came first?
More importantly, the use of adminq terminology here
is a going to be somewhat confusing with the unrelated
admin virtqueue just having landed in the virtio spec.
Is this terminology coming from some hardware spec?
This firmware adminq through the PCI interface is similar to how several
other network drivers have an adminq for more complex conversations with
the FW. Yes, I can see how having both this device adminq and the
virtio admin VQ in one place could be confusing, and we'll need to be
sure to be clear in the difference. This is part of why we put most of
the adminq cmd code in its own cmd.c file, away from the virtio/vdpa
handling in vdpa_dev.c.
Thanks,
sln
---
include/linux/pds/pds_adminq.h | 21 +
include/linux/pds/pds_common.h | 21 -
2 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/include/linux/pds/pds_adminq.h b/include/linux/pds/pds_adminq.h
index 98a60ce87b92..61b0a8634e1a 100644
--- a/include/linux/pds/pds_adminq.h
+++ b/include/linux/pds/pds_adminq.h
@@ -222,6 +222,27 @@ enum pds_core_lif_type {
PDS_CORE_LIF_TYPE_DEFAULT = 0,
};
+#define PDS_CORE_IFNAMSIZ16
+
+/**
+ * enum pds_core_logical_qtype - Logical Queue Types
+ * @PDS_CORE_QTYPE_ADMINQ:Administrative Queue
+ * @PDS_CORE_QTYPE_NOTIFYQ: Notify Queue
+ * @PDS_CORE_QTYPE_RXQ: Receive Queue
+ * @PDS_CORE_QTYPE_TXQ: Transmit Queue
+ * @PDS_CORE_QTYPE_EQ:Event Queue
+ * @PDS_CORE_QTYPE_MAX: Max queue type supported
+ */
+enum pds_core_logical_qtype {
+ PDS_CORE_QTYPE_ADMINQ = 0,
+ PDS_CORE_QTYPE_NOTIFYQ = 1,
+ PDS_CORE_QTYPE_RXQ = 2,
+ PDS_CORE_QTYPE_TXQ = 3,
+ PDS_CORE_QTYPE_EQ = 4,
+
+ PDS_CORE_QTYPE_MAX = 16 /* don't change - used in struct size */
+};
+
/**
* union pds_core_lif_config - LIF configuration
* @state: LIF state (enum pds_core_lif_state)
diff --git a/include/linux/pds/pds_common.h b/include/linux/pds/pds_common.h
index 2a0d1669cfd0..435c8e8161c2 100644
--- a/include/linux/pds/pds_common.h
+++ b/include/linux/pds/pds_common.h
@@ -41,27 +41,6 @@ enum pds_core_vif_types {
#define PDS_VDPA_DEV_NAMEPDS_CORE_DRV_NAME "." PDS_DEV_TYPE_VDPA_STR
-#define PDS_CORE_IFNAMSIZ16
-
-/**
- * enum pds_core_logical_qtype - Logical Queue Types
- * @PDS_CORE_QTYPE_ADMINQ:Administrative Queue
- * @PDS_CORE_QTYPE_NOTIFYQ: Notify Queue
- * @PDS_CORE_QTYPE_RXQ: Receive Queue
- * @PDS_CORE_QTYPE_TXQ: Transmit Queue
- * @PDS_CORE_QTYPE_EQ:Event Queue
- * @PDS_CORE_QTYPE_MAX: Max queue type supported
- */
-enum pds_core_logical_qtype {
- PDS_CORE_QTYPE_ADMINQ = 0,
- PDS_CORE_QTYPE_NOTIFYQ = 1,
- PDS_CORE_QTYPE_RXQ = 2,
- PDS_CORE_QTYPE_TXQ = 3,
- PDS_CORE_QTYPE_EQ = 4,
-
- PDS_CORE_QTYPE_MAX = 16 /* don't change - used in struct size */
-};
-
int pdsc_register_notify(struct notifier_block *nb);
void pdsc_unregister_notify(struct notifier_block *nb);
void *pdsc_get_pf_struct(struct pci_dev *vf_pdev);
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v6 virtio 11/11] pds_vdpa: pds_vdps.rst and Kconfig
Add the documentation and Kconfig entry for pds_vdpa driver.
Signed-off-by: Shannon Nelson
---
.../device_drivers/ethernet/amd/pds_vdpa.rst | 85 +++
.../device_drivers/ethernet/index.rst | 1 +
MAINTAINERS | 4 +
drivers/vdpa/Kconfig | 10 +++
4 files changed, 100 insertions(+)
create mode 100644
Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
diff --git a/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
b/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
new file mode 100644
index ..587927d3de92
--- /dev/null
+++ b/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
@@ -0,0 +1,85 @@
+.. SPDX-License-Identifier: GPL-2.0+
+.. note: can be edited and viewed with /usr/bin/formiko-vim
+
+==
+PCI vDPA driver for the AMD/Pensando(R) DSC adapter family
+==
+
+AMD/Pensando vDPA VF Device Driver
+
+Copyright(c) 2023 Advanced Micro Devices, Inc
+
+Overview
+
+
+The ``pds_vdpa`` driver is an auxiliary bus driver that supplies
+a vDPA device for use by the virtio network stack. It is used with
+the Pensando Virtual Function devices that offer vDPA and virtio queue
+services. It depends on the ``pds_core`` driver and hardware for the PF
+and VF PCI handling as well as for device configuration services.
+
+Using the device
+
+
+The ``pds_vdpa`` device is enabled via multiple configuration steps and
+depends on the ``pds_core`` driver to create and enable SR-IOV Virtual
+Function devices. After the VFs are enabled, we enable the vDPA service
+in the ``pds_core`` device to create the auxiliary devices used by pds_vdpa.
+
+Example steps:
+
+.. code-block:: bash
+
+ #!/bin/bash
+
+ modprobe pds_core
+ modprobe vdpa
+ modprobe pds_vdpa
+
+ PF_BDF=`ls /sys/module/pds_core/drivers/pci\:pds_core/*/sriov_numvfs | awk
-F / '{print $7}'`
+
+ # Enable vDPA VF auxiliary device(s) in the PF
+ devlink dev param set pci/$PF_BDF name enable_vnet cmode runtime value true
+
+ # Create a VF for vDPA use
+ echo 1 > /sys/bus/pci/drivers/pds_core/$PF_BDF/sriov_numvfs
+
+ # Find the vDPA services/devices available
+ PDS_VDPA_MGMT=`vdpa mgmtdev show | grep vDPA | head -1 | cut -d: -f1`
+
+ # Create a vDPA device for use in virtio network configurations
+ vdpa dev add name vdpa1 mgmtdev $PDS_VDPA_MGMT mac 00:11:22:33:44:55
+
+ # Set up an ethernet interface on the vdpa device
+ modprobe virtio_vdpa
+
+
+
+Enabling the driver
+===
+
+The driver is enabled via the standard kernel configuration system,
+using the make command::
+
+ make oldconfig/menuconfig/etc.
+
+The driver is located in the menu structure at:
+
+ -> Device Drivers
+-> Network device support (NETDEVICES [=y])
+ -> Ethernet driver support
+-> Pensando devices
+ -> Pensando Ethernet PDS_VDPA Support
+
+Support
+===
+
+For general Linux networking support, please use the netdev mailing
+list, which is monitored by Pensando personnel::
+
+ net...@vger.kernel.org
+
+For more specific support needs, please use the Pensando driver support
+email::
+
+ driv...@pensando.io
diff --git a/Documentation/networking/device_drivers/ethernet/index.rst
b/Documentation/networking/device_drivers/ethernet/index.rst
index 417ca514a4d0..94ecb67c0885 100644
--- a/Documentation/networking/device_drivers/ethernet/index.rst
+++ b/Documentation/networking/device_drivers/ethernet/index.rst
@@ -15,6 +15,7 @@ Contents:
amazon/ena
altera/altera_tse
amd/pds_core
+ amd/pds_vdpa
aquantia/atlantic
chelsio/cxgb
cirrus/cs89x0
diff --git a/MAINTAINERS b/MAINTAINERS
index e2fd64c2ebdc..c3f509eeaf1d 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -22296,6 +22296,10 @@ F: include/linux/vringh.h
F: include/uapi/linux/virtio_*.h
F: tools/virtio/
+PDS DSC VIRTIO DATA PATH ACCELERATOR
+R: Shannon Nelson
+F: drivers/vdpa/pds/
+
VIRTIO CRYPTO DRIVER
M: Gonglei
L: virtualization@lists.linux-foundation.org
diff --git a/drivers/vdpa/Kconfig b/drivers/vdpa/Kconfig
index cd6ad92f3f05..656c1cb541de 100644
--- a/drivers/vdpa/Kconfig
+++ b/drivers/vdpa/Kconfig
@@ -116,4 +116,14 @@ config ALIBABA_ENI_VDPA
This driver includes a HW monitor device that
reads health values from the DPU.
+config PDS_VDPA
+ tristate "vDPA driver for AMD/Pensando DSC devices"
+ select VIRTIO_PCI_LIB
+ depends on PCI_MSI
+ depends on PDS_CORE
+ help
+ vDPA network driver for AMD/Pensando's PDS Core devices.
+ With this driver, the VirtIO dataplane can be
+ offloaded to an AMD/Pensando DSC device.
+
endif # VDPA
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
[PATCH v6 virtio 09/11] pds_vdpa: add support for vdpa and vdpamgmt interfaces
This is the vDPA device support, where we advertise that we can
support the virtio queues and deal with the configuration work
through the pds_core's adminq.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/pds/aux_drv.c | 15 +
drivers/vdpa/pds/aux_drv.h | 1 +
drivers/vdpa/pds/debugfs.c | 263
drivers/vdpa/pds/debugfs.h | 5 +
drivers/vdpa/pds/vdpa_dev.c | 605 +++-
drivers/vdpa/pds/vdpa_dev.h | 4 +-
6 files changed, 891 insertions(+), 2 deletions(-)
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
index 0c4a135b1484..186e9ee22eb1 100644
--- a/drivers/vdpa/pds/aux_drv.c
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -63,8 +63,21 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
goto err_free_mgmt_info;
}
+ /* Let vdpa know that we can provide devices */
+ err = vdpa_mgmtdev_register(_aux->vdpa_mdev);
+ if (err) {
+ dev_err(dev, "%s: Failed to initialize vdpa_mgmt interface:
%pe\n",
+ __func__, ERR_PTR(err));
+ goto err_free_virtio;
+ }
+
+ pds_vdpa_debugfs_add_pcidev(vdpa_aux);
+ pds_vdpa_debugfs_add_ident(vdpa_aux);
+
return 0;
+err_free_virtio:
+ vp_modern_remove(_aux->vd_mdev);
err_free_mgmt_info:
pci_free_irq_vectors(padev->vf_pdev);
err_free_mem:
@@ -79,9 +92,11 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
struct device *dev = _dev->dev;
+ vdpa_mgmtdev_unregister(_aux->vdpa_mdev);
vp_modern_remove(_aux->vd_mdev);
pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
+ pds_vdpa_debugfs_del_vdpadev(vdpa_aux);
kfree(vdpa_aux);
auxiliary_set_drvdata(aux_dev, NULL);
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
index 99e0ff340bfa..26b75344156e 100644
--- a/drivers/vdpa/pds/aux_drv.h
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -13,6 +13,7 @@ struct pds_vdpa_aux {
struct pds_auxiliary_dev *padev;
struct vdpa_mgmt_dev vdpa_mdev;
+ struct pds_vdpa_device *pdsv;
struct pds_vdpa_ident ident;
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
index d91dceb07380..21a0dc0cb607 100644
--- a/drivers/vdpa/pds/debugfs.c
+++ b/drivers/vdpa/pds/debugfs.c
@@ -10,6 +10,7 @@
#include
#include "aux_drv.h"
+#include "vdpa_dev.h"
#include "debugfs.h"
static struct dentry *dbfs_dir;
@@ -24,3 +25,265 @@ void pds_vdpa_debugfs_destroy(void)
debugfs_remove_recursive(dbfs_dir);
dbfs_dir = NULL;
}
+
+#define PRINT_SBIT_NAME(__seq, __f, __name) \
+ do {\
+ if ((__f) & (__name)) \
+ seq_printf(__seq, " %s",
[PATCH v6 virtio 10/11] pds_vdpa: subscribe to the pds_core events
Register for the pds_core's notification events, primarily to
find out when the FW has been reset so we can pass this on
back up the chain.
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/vdpa_dev.c | 59 -
drivers/vdpa/pds/vdpa_dev.h | 1 +
2 files changed, 59 insertions(+), 1 deletion(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index 07b98dff5701..9afa803c4f21 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -23,6 +23,52 @@ static struct pds_vdpa_device *vdpa_to_pdsv(struct
vdpa_device *vdpa_dev)
return container_of(vdpa_dev, struct pds_vdpa_device, vdpa_dev);
}
+static int pds_vdpa_notify_handler(struct notifier_block *nb,
+ unsigned long ecode,
+ void *data)
+{
+ struct pds_vdpa_device *pdsv = container_of(nb, struct pds_vdpa_device,
nb);
+ struct device *dev = >vdpa_aux->padev->aux_dev.dev;
+
+ dev_dbg(dev, "%s: event code %lu\n", __func__, ecode);
+
+ if (ecode == PDS_EVENT_RESET || ecode == PDS_EVENT_LINK_CHANGE) {
+ if (pdsv->config_cb.callback)
+ pdsv->config_cb.callback(pdsv->config_cb.private);
+ }
+
+ return 0;
+}
+
+static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv)
+{
+ struct device *dev = >vdpa_aux->padev->aux_dev.dev;
+ struct notifier_block *nb = >nb;
+ int err;
+
+ if (!nb->notifier_call) {
+ nb->notifier_call = pds_vdpa_notify_handler;
+ err = pdsc_register_notify(nb);
+ if (err) {
+ nb->notifier_call = NULL;
+ dev_err(dev, "failed to register pds event handler:
%ps\n",
+ ERR_PTR(err));
+ return -EINVAL;
+ }
+ dev_dbg(dev, "pds event handler registered\n");
+ }
+
+ return 0;
+}
+
+static void pds_vdpa_unregister_event_handler(struct pds_vdpa_device *pdsv)
+{
+ if (pdsv->nb.notifier_call) {
+ pdsc_unregister_notify(>nb);
+ pdsv->nb.notifier_call = NULL;
+ }
+}
+
static int pds_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
u64 desc_addr, u64 driver_addr, u64
device_addr)
{
@@ -594,6 +640,12 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
pdsv->vdpa_dev.mdev = _aux->vdpa_mdev;
+ err = pds_vdpa_register_event_handler(pdsv);
+ if (err) {
+ dev_err(dev, "Failed to register for PDS events: %pe\n",
ERR_PTR(err));
+ goto err_unmap;
+ }
+
/* We use the _vdpa_register_device() call rather than the
* vdpa_register_device() to avoid a deadlock because our
* dev_add() is called with the vdpa_dev_lock already set
@@ -602,13 +654,15 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
err = _vdpa_register_device(>vdpa_dev, pdsv->num_vqs);
if (err) {
dev_err(dev, "Failed to register to vDPA bus: %pe\n",
ERR_PTR(err));
- goto err_unmap;
+ goto err_unevent;
}
pds_vdpa_debugfs_add_vdpadev(vdpa_aux);
return 0;
+err_unevent:
+ pds_vdpa_unregister_event_handler(pdsv);
err_unmap:
put_device(>vdpa_dev.dev);
vdpa_aux->pdsv = NULL;
@@ -618,8 +672,11 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
struct vdpa_device *vdpa_dev)
{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
struct pds_vdpa_aux *vdpa_aux;
+ pds_vdpa_unregister_event_handler(pdsv);
+
vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
_vdpa_unregister_device(vdpa_dev);
diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h
index 25c1d192f0ef..a1bc37de9537 100644
--- a/drivers/vdpa/pds/vdpa_dev.h
+++ b/drivers/vdpa/pds/vdpa_dev.h
@@ -40,6 +40,7 @@ struct pds_vdpa_device {
u8 vdpa_index; /* rsvd for future subdevice use */
u8 num_vqs; /* num vqs in use */
struct vdpa_callback config_cb;
+ struct notifier_block nb;
};
#define PDS_VDPA_PACKED_INVERT_IDX 0x8000
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v6 virtio 03/11] pds_vdpa: Add new vDPA driver for AMD/Pensando DSC
This is the initial auxiliary driver framework for a new vDPA
device driver, an auxiliary_bus client of the pds_core driver.
The pds_core driver supplies the PCI services for the VF device
and for accessing the adminq in the PF device.
This patch adds the very basics of registering for the auxiliary
device and setting up debugfs entries.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/Makefile | 1 +
drivers/vdpa/pds/Makefile | 8
drivers/vdpa/pds/aux_drv.c | 83 ++
drivers/vdpa/pds/aux_drv.h | 15 ++
drivers/vdpa/pds/debugfs.c | 25 ++
drivers/vdpa/pds/debugfs.h | 12 +
include/linux/pds/pds_common.h | 2 +
7 files changed, 146 insertions(+)
create mode 100644 drivers/vdpa/pds/Makefile
create mode 100644 drivers/vdpa/pds/aux_drv.c
create mode 100644 drivers/vdpa/pds/aux_drv.h
create mode 100644 drivers/vdpa/pds/debugfs.c
create mode 100644 drivers/vdpa/pds/debugfs.h
diff --git a/drivers/vdpa/Makefile b/drivers/vdpa/Makefile
index 59396ff2a318..8f53c6f3cca7 100644
--- a/drivers/vdpa/Makefile
+++ b/drivers/vdpa/Makefile
@@ -7,3 +7,4 @@ obj-$(CONFIG_MLX5_VDPA) += mlx5/
obj-$(CONFIG_VP_VDPA)+= virtio_pci/
obj-$(CONFIG_ALIBABA_ENI_VDPA) += alibaba/
obj-$(CONFIG_SNET_VDPA) += solidrun/
+obj-$(CONFIG_PDS_VDPA) += pds/
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
new file mode 100644
index ..a9cd2f450ae1
--- /dev/null
+++ b/drivers/vdpa/pds/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# Copyright(c) 2023 Advanced Micro Devices, Inc
+
+obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
+
+pds_vdpa-y := aux_drv.o
+
+pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
new file mode 100644
index ..e4a0ad61ea22
--- /dev/null
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include "aux_drv.h"
+#include "debugfs.h"
+
+static const struct auxiliary_device_id pds_vdpa_id_table[] = {
+ { .name = PDS_VDPA_DEV_NAME, },
+ {},
+};
+
+static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
+ const struct auxiliary_device_id *id)
+
+{
+ struct pds_auxiliary_dev *padev =
+ container_of(aux_dev, struct pds_auxiliary_dev, aux_dev);
+ struct pds_vdpa_aux *vdpa_aux;
+
+ vdpa_aux = kzalloc(sizeof(*vdpa_aux), GFP_KERNEL);
+ if (!vdpa_aux)
+ return -ENOMEM;
+
+ vdpa_aux->padev = padev;
+ auxiliary_set_drvdata(aux_dev, vdpa_aux);
+
+ return 0;
+}
+
+static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
+{
+ struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
+ struct device *dev = _dev->dev;
+
+ kfree(vdpa_aux);
+ auxiliary_set_drvdata(aux_dev, NULL);
+
+ dev_info(dev, "Removed\n");
+}
+
+static struct auxiliary_driver pds_vdpa_driver = {
+ .name = PDS_DEV_TYPE_VDPA_STR,
+ .probe = pds_vdpa_probe,
+ .remove = pds_vdpa_remove,
+ .id_table = pds_vdpa_id_table,
+};
+
+static void __exit pds_vdpa_cleanup(void)
+{
+ auxiliary_driver_unregister(_vdpa_driver);
+
+ pds_vdpa_debugfs_destroy();
+}
+module_exit(pds_vdpa_cleanup);
+
+static int __init pds_vdpa_init(void)
+{
+ int err;
+
+ pds_vdpa_debugfs_create();
+
+ err = auxiliary_driver_register(_vdpa_driver);
+ if (err) {
+ pr_err("%s: aux driver register failed: %pe\n",
+ PDS_VDPA_DRV_NAME, ERR_PTR(err));
+ pds_vdpa_debugfs_destroy();
+ }
+
+ return err;
+}
+module_init(pds_vdpa_init);
+
+MODULE_DESCRIPTION(PDS_VDPA_DRV_DESCRIPTION);
+MODULE_AUTHOR("Advanced Micro Devices, Inc");
+MODULE_LICENSE("GPL");
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
new file mode 100644
index ..f1e99359424e
--- /dev/null
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#ifndef _AUX_DRV_H_
+#define _AUX_DRV_H_
+
+#define PDS_VDPA_DRV_DESCRIPTION"AMD/Pensando vDPA VF Device Driver"
+#define PDS_VDPA_DRV_NAME KBUILD_MODNAME
+
+struct pds_vdpa_aux {
+ struct pds_auxiliary_dev *padev;
+
+ struct dentry *dentry;
+};
+#endif /* _AUX_DRV_H_ */
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
new file mode 100644
index ..5be22fb7a76a
--- /dev/null
+++ b/drivers/vdpa/pds/debugfs.c
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+
+#include
+#include
+#include
+#include
+
+#include "aux_drv.h"
+#include "debugfs.h"
+
+static struct dentry *dbfs_dir;
+
+void
[PATCH v6 virtio 05/11] pds_vdpa: new adminq entries
Add new adminq definitions in support for vDPA operations.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
include/linux/pds/pds_adminq.h | 226 +
1 file changed, 226 insertions(+)
diff --git a/include/linux/pds/pds_adminq.h b/include/linux/pds/pds_adminq.h
index 61b0a8634e1a..bcba7fda3cc9 100644
--- a/include/linux/pds/pds_adminq.h
+++ b/include/linux/pds/pds_adminq.h
@@ -605,6 +605,219 @@ struct pds_core_q_init_comp {
u8 color;
};
+/*
+ * enum pds_vdpa_cmd_opcode - vDPA Device commands
+ */
+enum pds_vdpa_cmd_opcode {
+ PDS_VDPA_CMD_INIT = 48,
+ PDS_VDPA_CMD_IDENT = 49,
+ PDS_VDPA_CMD_RESET = 51,
+ PDS_VDPA_CMD_VQ_RESET = 52,
+ PDS_VDPA_CMD_VQ_INIT= 53,
+ PDS_VDPA_CMD_STATUS_UPDATE = 54,
+ PDS_VDPA_CMD_SET_FEATURES = 55,
+ PDS_VDPA_CMD_SET_ATTR = 56,
+};
+
+/**
+ * struct pds_vdpa_cmd - generic command
+ * @opcode:Opcode
+ * @vdpa_index:Index for vdpa subdevice
+ * @vf_id: VF id
+ */
+struct pds_vdpa_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+};
+
+/**
+ * struct pds_vdpa_init_cmd - INIT command
+ * @opcode:Opcode PDS_VDPA_CMD_INIT
+ * @vdpa_index: Index for vdpa subdevice
+ * @vf_id: VF id
+ */
+struct pds_vdpa_init_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+};
+
+/**
+ * struct pds_vdpa_ident - vDPA identification data
+ * @hw_features: vDPA features supported by device
+ * @max_vqs: max queues available (2 queues for a single queuepair)
+ * @max_qlen: log(2) of maximum number of descriptors
+ * @min_qlen: log(2) of minimum number of descriptors
+ *
+ * This struct is used in a DMA block that is set up for the PDS_VDPA_CMD_IDENT
+ * transaction. Set up the DMA block and send the address in the IDENT cmd
+ * data, the DSC will write the ident information, then we can remove the DMA
+ * block after reading the answer. If the completion status is 0, then there
+ * is valid information, else there was an error and the data should be
invalid.
+ */
+struct pds_vdpa_ident {
+ __le64 hw_features;
+ __le16 max_vqs;
+ __le16 max_qlen;
+ __le16 min_qlen;
+};
+
+/**
+ * struct pds_vdpa_ident_cmd - IDENT command
+ * @opcode:Opcode PDS_VDPA_CMD_IDENT
+ * @rsvd: Word boundary padding
+ * @vf_id: VF id
+ * @len: length of ident info DMA space
+ * @ident_pa: address for DMA of ident info (struct pds_vdpa_ident)
+ * only used for this transaction, then forgotten by DSC
+ */
+struct pds_vdpa_ident_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 vf_id;
+ __le32 len;
+ __le64 ident_pa;
+};
+
+/**
+ * struct pds_vdpa_status_cmd - STATUS_UPDATE command
+ * @opcode:Opcode PDS_VDPA_CMD_STATUS_UPDATE
+ * @vdpa_index: Index for vdpa subdevice
+ * @vf_id: VF id
+ * @status:new status bits
+ */
+struct pds_vdpa_status_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+ u8 status;
+};
+
+/**
+ * enum pds_vdpa_attr - List of VDPA device attributes
+ * @PDS_VDPA_ATTR_MAC: MAC address
+ * @PDS_VDPA_ATTR_MAX_VQ_PAIRS: Max virtqueue pairs
+ */
+enum pds_vdpa_attr {
+ PDS_VDPA_ATTR_MAC = 1,
+ PDS_VDPA_ATTR_MAX_VQ_PAIRS = 2,
+};
+
+/**
+ * struct pds_vdpa_setattr_cmd - SET_ATTR command
+ * @opcode:Opcode PDS_VDPA_CMD_SET_ATTR
+ * @vdpa_index:Index for vdpa subdevice
+ * @vf_id: VF id
+ * @attr: attribute to be changed (enum pds_vdpa_attr)
+ * @pad: Word boundary padding
+ * @mac: new mac address to be assigned as vdpa device address
+ * @max_vq_pairs: new limit of virtqueue pairs
+ */
+struct pds_vdpa_setattr_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+ u8 attr;
+ u8 pad[3];
+ union {
+ u8 mac[6];
+ __le16 max_vq_pairs;
+ } __packed;
+};
+
+/**
+ * struct pds_vdpa_vq_init_cmd - queue init command
+ * @opcode: Opcode PDS_VDPA_CMD_VQ_INIT
+ * @vdpa_index:Index for vdpa subdevice
+ * @vf_id: VF id
+ * @qid: Queue id (bit0 clear = rx, bit0 set = tx, qid=N is ctrlq)
+ * @len: log(2) of max descriptor count
+ * @desc_addr: DMA address of descriptor area
+ * @avail_addr:DMA address of available descriptors (aka driver area)
+ * @used_addr: DMA address of used descriptors (aka device area)
+ * @intr_index:interrupt index
+ * @avail_index: initial device position in available ring
+ * @used_index:initial device position in used ring
+ */
+struct pds_vdpa_vq_init_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+ __le16 qid;
+ __le16 len;
+ __le64 desc_addr;
+ __le64
[PATCH v6 virtio 08/11] pds_vdpa: add vdpa config client commands
These are the adminq commands that will be needed for
setting up and using the vDPA device. There are a number
of commands defined in the FW's API, but by making use of
the FW's virtio BAR we only need a few of these commands
for vDPA support.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/pds/Makefile | 1 +
drivers/vdpa/pds/cmds.c | 185
drivers/vdpa/pds/cmds.h | 18
drivers/vdpa/pds/vdpa_dev.h | 33 ++-
4 files changed, 236 insertions(+), 1 deletion(-)
create mode 100644 drivers/vdpa/pds/cmds.c
create mode 100644 drivers/vdpa/pds/cmds.h
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
index 13b50394ec64..2e22418e3ab3 100644
--- a/drivers/vdpa/pds/Makefile
+++ b/drivers/vdpa/pds/Makefile
@@ -4,6 +4,7 @@
obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
pds_vdpa-y := aux_drv.o \
+ cmds.o \
vdpa_dev.o
pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/vdpa/pds/cmds.c b/drivers/vdpa/pds/cmds.c
new file mode 100644
index ..80863a41c3cd
--- /dev/null
+++ b/drivers/vdpa/pds/cmds.c
@@ -0,0 +1,185 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include "vdpa_dev.h"
+#include "aux_drv.h"
+#include "cmds.h"
+
+int pds_vdpa_init_hw(struct pds_vdpa_device *pdsv)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_init.opcode = PDS_VDPA_CMD_INIT,
+ .vdpa_init.vdpa_index = pdsv->vdpa_index,
+ .vdpa_init.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ /* Initialize the vdpa/virtio device */
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa_init),
+ , 0);
+ if (err)
+ dev_dbg(dev, "Failed to init hw, status %d: %pe\n",
+ comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_reset(struct pds_vdpa_device *pdsv)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa.opcode = PDS_VDPA_CMD_RESET,
+ .vdpa.vdpa_index = pdsv->vdpa_index,
+ .vdpa.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa), , 0);
+ if (err)
+ dev_dbg(dev, "Failed to reset hw, status %d: %pe\n",
+ comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_set_status(struct pds_vdpa_device *pdsv, u8 status)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_status.opcode = PDS_VDPA_CMD_STATUS_UPDATE,
+ .vdpa_status.vdpa_index = pdsv->vdpa_index,
+ .vdpa_status.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ .vdpa_status.status = status,
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa_status),
, 0);
+ if (err)
+ dev_dbg(dev, "Failed to set status to %#x, error status %d:
%pe\n",
+ status, comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_set_mac(struct pds_vdpa_device *pdsv, u8 *mac)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR,
+ .vdpa_setattr.vdpa_index = pdsv->vdpa_index,
+ .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ .vdpa_setattr.attr = PDS_VDPA_ATTR_MAC,
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ ether_addr_copy(cmd.vdpa_setattr.mac, mac);
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa_setattr),
+ , 0);
+ if (err)
+ dev_dbg(dev, "Failed to set mac address %pM, status %d: %pe\n",
+ mac, comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_set_max_vq_pairs(struct pds_vdpa_device *pdsv, u16 max_vqp)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR,
+ .vdpa_setattr.vdpa_index = pdsv->vdpa_index,
+ .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
[PATCH v6 virtio 07/11] pds_vdpa: virtio bar setup for vdpa
Prep and use the "modern" virtio bar utilities to get our
virtio config space ready.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/pds/aux_drv.c | 25 +
drivers/vdpa/pds/aux_drv.h | 3 +++
2 files changed, 28 insertions(+)
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
index aa748cf55d2b..0c4a135b1484 100644
--- a/drivers/vdpa/pds/aux_drv.c
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -4,6 +4,7 @@
#include
#include
#include
+#include
#include
#include
@@ -19,12 +20,22 @@ static const struct auxiliary_device_id pds_vdpa_id_table[]
= {
{},
};
+static int pds_vdpa_device_id_check(struct pci_dev *pdev)
+{
+ if (pdev->device != PCI_DEVICE_ID_PENSANDO_VDPA_VF ||
+ pdev->vendor != PCI_VENDOR_ID_PENSANDO)
+ return -ENODEV;
+
+ return PCI_DEVICE_ID_PENSANDO_VDPA_VF;
+}
+
static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
const struct auxiliary_device_id *id)
{
struct pds_auxiliary_dev *padev =
container_of(aux_dev, struct pds_auxiliary_dev, aux_dev);
+ struct device *dev = _dev->dev;
struct pds_vdpa_aux *vdpa_aux;
int err;
@@ -41,8 +52,21 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
if (err)
goto err_free_mem;
+ /* Find the virtio configuration */
+ vdpa_aux->vd_mdev.pci_dev = padev->vf_pdev;
+ vdpa_aux->vd_mdev.device_id_check = pds_vdpa_device_id_check;
+ vdpa_aux->vd_mdev.dma_mask = DMA_BIT_MASK(PDS_CORE_ADDR_LEN);
+ err = vp_modern_probe(_aux->vd_mdev);
+ if (err) {
+ dev_err(dev, "Unable to probe for virtio configuration: %pe\n",
+ ERR_PTR(err));
+ goto err_free_mgmt_info;
+ }
+
return 0;
+err_free_mgmt_info:
+ pci_free_irq_vectors(padev->vf_pdev);
err_free_mem:
kfree(vdpa_aux);
auxiliary_set_drvdata(aux_dev, NULL);
@@ -55,6 +79,7 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
struct device *dev = _dev->dev;
+ vp_modern_remove(_aux->vd_mdev);
pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
kfree(vdpa_aux);
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
index dcec782e79eb..99e0ff340bfa 100644
--- a/drivers/vdpa/pds/aux_drv.h
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -4,6 +4,8 @@
#ifndef _AUX_DRV_H_
#define _AUX_DRV_H_
+#include
+
#define PDS_VDPA_DRV_DESCRIPTION"AMD/Pensando vDPA VF Device Driver"
#define PDS_VDPA_DRV_NAME KBUILD_MODNAME
@@ -16,6 +18,7 @@ struct pds_vdpa_aux {
int vf_id;
struct dentry *dentry;
+ struct virtio_pci_modern_device vd_mdev;
int nintrs;
};
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v6 virtio 06/11] pds_vdpa: get vdpa management info
Find the vDPA management information from the DSC in order to
advertise it to the vdpa subsystem.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/pds/Makefile | 3 +-
drivers/vdpa/pds/aux_drv.c | 17 ++
drivers/vdpa/pds/aux_drv.h | 7 +++
drivers/vdpa/pds/debugfs.c | 1 +
drivers/vdpa/pds/vdpa_dev.c | 108
drivers/vdpa/pds/vdpa_dev.h | 15 +
6 files changed, 150 insertions(+), 1 deletion(-)
create mode 100644 drivers/vdpa/pds/vdpa_dev.c
create mode 100644 drivers/vdpa/pds/vdpa_dev.h
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
index a9cd2f450ae1..13b50394ec64 100644
--- a/drivers/vdpa/pds/Makefile
+++ b/drivers/vdpa/pds/Makefile
@@ -3,6 +3,7 @@
obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
-pds_vdpa-y := aux_drv.o
+pds_vdpa-y := aux_drv.o \
+ vdpa_dev.o
pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
index e4a0ad61ea22..aa748cf55d2b 100644
--- a/drivers/vdpa/pds/aux_drv.c
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -3,6 +3,7 @@
#include
#include
+#include
#include
#include
@@ -11,6 +12,7 @@
#include "aux_drv.h"
#include "debugfs.h"
+#include "vdpa_dev.h"
static const struct auxiliary_device_id pds_vdpa_id_table[] = {
{ .name = PDS_VDPA_DEV_NAME, },
@@ -24,15 +26,28 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
struct pds_auxiliary_dev *padev =
container_of(aux_dev, struct pds_auxiliary_dev, aux_dev);
struct pds_vdpa_aux *vdpa_aux;
+ int err;
vdpa_aux = kzalloc(sizeof(*vdpa_aux), GFP_KERNEL);
if (!vdpa_aux)
return -ENOMEM;
vdpa_aux->padev = padev;
+ vdpa_aux->vf_id = pci_iov_vf_id(padev->vf_pdev);
auxiliary_set_drvdata(aux_dev, vdpa_aux);
+ /* Get device ident info and set up the vdpa_mgmt_dev */
+ err = pds_vdpa_get_mgmt_info(vdpa_aux);
+ if (err)
+ goto err_free_mem;
+
return 0;
+
+err_free_mem:
+ kfree(vdpa_aux);
+ auxiliary_set_drvdata(aux_dev, NULL);
+
+ return err;
}
static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
@@ -40,6 +55,8 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
struct device *dev = _dev->dev;
+ pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
+
kfree(vdpa_aux);
auxiliary_set_drvdata(aux_dev, NULL);
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
index f1e99359424e..dcec782e79eb 100644
--- a/drivers/vdpa/pds/aux_drv.h
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -10,6 +10,13 @@
struct pds_vdpa_aux {
struct pds_auxiliary_dev *padev;
+ struct vdpa_mgmt_dev vdpa_mdev;
+
+ struct pds_vdpa_ident ident;
+
+ int vf_id;
struct dentry *dentry;
+
+ int nintrs;
};
#endif /* _AUX_DRV_H_ */
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
index 5be22fb7a76a..d91dceb07380 100644
--- a/drivers/vdpa/pds/debugfs.c
+++ b/drivers/vdpa/pds/debugfs.c
@@ -2,6 +2,7 @@
/* Copyright(c) 2023 Advanced Micro Devices, Inc */
#include
+#include
#include
#include
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
new file mode 100644
index ..0f0f0ab8b811
--- /dev/null
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include "vdpa_dev.h"
+#include "aux_drv.h"
+
+static struct virtio_device_id pds_vdpa_id_table[] = {
+ {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
+ {0},
+};
+
+static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
+ const struct vdpa_dev_set_config *add_config)
+{
+ return -EOPNOTSUPP;
+}
+
+static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
+struct vdpa_device *vdpa_dev)
+{
+}
+
+static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = {
+ .dev_add = pds_vdpa_dev_add,
+ .dev_del = pds_vdpa_dev_del
+};
+
+int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
+{
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_ident.opcode = PDS_VDPA_CMD_IDENT,
+ .vdpa_ident.vf_id = cpu_to_le16(vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ struct vdpa_mgmt_dev *mgmt;
+ struct pci_dev *pf_pdev;
+ struct device *pf_dev;
+ struct pci_dev *pdev;
+ dma_addr_t ident_pa;
+ struct device *dev;
+ u16 dev_intrs;
+ u16 max_vqs;
+ int err;
+
+ dev = _aux->padev->aux_dev.dev;
+ pdev = vdpa_aux->padev->vf_pdev;
+ mgmt = _aux->vdpa_mdev;
+
+ /* Get resource info through the PF's adminq. It
[PATCH v6 virtio 04/11] pds_vdpa: move enum from common to adminq header
The pds_core_logical_qtype enum and IFNAMSIZ are not needed
in the common PDS header, only needed when working with the
adminq, so move them to the adminq header.
Note: This patch might conflict with pds_vfio patches that are
in review, depending on which patchset gets pulled first.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
include/linux/pds/pds_adminq.h | 21 +
include/linux/pds/pds_common.h | 21 -
2 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/include/linux/pds/pds_adminq.h b/include/linux/pds/pds_adminq.h
index 98a60ce87b92..61b0a8634e1a 100644
--- a/include/linux/pds/pds_adminq.h
+++ b/include/linux/pds/pds_adminq.h
@@ -222,6 +222,27 @@ enum pds_core_lif_type {
PDS_CORE_LIF_TYPE_DEFAULT = 0,
};
+#define PDS_CORE_IFNAMSIZ 16
+
+/**
+ * enum pds_core_logical_qtype - Logical Queue Types
+ * @PDS_CORE_QTYPE_ADMINQ:Administrative Queue
+ * @PDS_CORE_QTYPE_NOTIFYQ: Notify Queue
+ * @PDS_CORE_QTYPE_RXQ: Receive Queue
+ * @PDS_CORE_QTYPE_TXQ: Transmit Queue
+ * @PDS_CORE_QTYPE_EQ:Event Queue
+ * @PDS_CORE_QTYPE_MAX: Max queue type supported
+ */
+enum pds_core_logical_qtype {
+ PDS_CORE_QTYPE_ADMINQ = 0,
+ PDS_CORE_QTYPE_NOTIFYQ = 1,
+ PDS_CORE_QTYPE_RXQ = 2,
+ PDS_CORE_QTYPE_TXQ = 3,
+ PDS_CORE_QTYPE_EQ = 4,
+
+ PDS_CORE_QTYPE_MAX = 16 /* don't change - used in struct size */
+};
+
/**
* union pds_core_lif_config - LIF configuration
* @state: LIF state (enum pds_core_lif_state)
diff --git a/include/linux/pds/pds_common.h b/include/linux/pds/pds_common.h
index 2a0d1669cfd0..435c8e8161c2 100644
--- a/include/linux/pds/pds_common.h
+++ b/include/linux/pds/pds_common.h
@@ -41,27 +41,6 @@ enum pds_core_vif_types {
#define PDS_VDPA_DEV_NAME PDS_CORE_DRV_NAME "." PDS_DEV_TYPE_VDPA_STR
-#define PDS_CORE_IFNAMSIZ 16
-
-/**
- * enum pds_core_logical_qtype - Logical Queue Types
- * @PDS_CORE_QTYPE_ADMINQ:Administrative Queue
- * @PDS_CORE_QTYPE_NOTIFYQ: Notify Queue
- * @PDS_CORE_QTYPE_RXQ: Receive Queue
- * @PDS_CORE_QTYPE_TXQ: Transmit Queue
- * @PDS_CORE_QTYPE_EQ:Event Queue
- * @PDS_CORE_QTYPE_MAX: Max queue type supported
- */
-enum pds_core_logical_qtype {
- PDS_CORE_QTYPE_ADMINQ = 0,
- PDS_CORE_QTYPE_NOTIFYQ = 1,
- PDS_CORE_QTYPE_RXQ = 2,
- PDS_CORE_QTYPE_TXQ = 3,
- PDS_CORE_QTYPE_EQ = 4,
-
- PDS_CORE_QTYPE_MAX = 16 /* don't change - used in struct size */
-};
-
int pdsc_register_notify(struct notifier_block *nb);
void pdsc_unregister_notify(struct notifier_block *nb);
void *pdsc_get_pf_struct(struct pci_dev *vf_pdev);
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v6 virtio 00/11] pds_vdpa driver
This patchset implements a new module for the AMD/Pensando DSC that supports vDPA services on PDS Core VF devices. This code is based on and depends on include files from the pds_core driver described here[0]. The pds_core driver creates the auxiliary_bus devices that this module connects to, and this creates vdpa devices for use by the vdpa module. The first version of this driver was a part of the original pds_core RFC [1] but has since been reworked to pull out the PCI driver and to make better use of the virtio and virtio_net configuration spaces made available by the DSC's PCI configuration. As the device development has progressed, the ability to rely on the virtio config spaces has grown. This patchset includes a modification to the existing vp_modern_probe() which implements overrides for the PCI device id check and the DMA mask. These are intended to be used with vendor vDPA devices that implement enough of the virtio config space to be used directly, but don't use the virtio device id. To use this module, enable the VFs and turn on the vDPA services in the pds_core PF, then use the 'vdpa' utility to create devices for use by virtio_vdpa or vhost_vdpa: echo 1 > /sys/bus/pci/drivers/pds_core/$PF_BDF/sriov_numvfs devlink dev param set pci/$PF_BDF name enable_vnet value true cmode runtime PDS_VDPA_MGMT=`vdpa mgmtdev show | grep vDPA | head -1 | cut -d: -f1` vdpa dev add name vdpa1 mgmtdev $PDS_VDPA_MGMT mac 00:11:22:33:44:55 [0] Link: https://lore.kernel.org/netdev/20230419170427.1108-1-shannon.nel...@amd.com/ [1] Link: https://lore.kernel.org/netdev/20221118225656.48309-1-snel...@pensando.io/ Changes: v6: - removed misleading comments from pds_vdpa_notify_handler() - added Kconfig "select VIRTIO_PCI_LIB" and "depends on PCI_MSI" - changed pds_vdpa_set_status() to use an adminq cmd to bypass a FW sequencing issue - added map vq notify after features_ok for correct placement after feature negotiation - added support for dev_add() option VDPA_ATTR_DEV_FEATURES - reworked get/set_vq_state() to better support packed, split, and legacy use - dropped Jason's Acked-by on 09/11 because of these last four changes v5: Link: https://lore.kernel.org/virtualization/20230503181240.14009-1-shannon.nel...@amd.com/ - split dma_mask and device_id_check() into separate patches - simplify use of dma_mask into a single line change - changed test of VIRTIO_F_RING_PACKED to use BIT_ULL() v4: Link: https://lore.kernel.org/virtualization/20230425212602.1157-1-shannon.nel...@amd.com/ - rename device_id_check_override() to device_id_check() - make device_id_check() return the device_id found and checked - removed pds_vdpa.h, put its adminq changes into pds_adminq.h - added a patch to separate out the adminq changes - added a patch to move an adminq enum from pds_common.h to pds_adminq.h - moved adminq calls for get/set_vq_state into cmds.c - limit max_vqs by number of msix available - don't increment nintrs for CVQ, it should already be covered from max_vqs - pds_core API related rework following pds_core inclusion to net-next - use non-debugfs method to find PF pci address in pds_vdpa.rst instructions v3: Link: https://lore.kernel.org/netdev/20230330192313.62018-1-shannon.nel...@amd.com/ - added a patch to modify vp_modern_probe() such that specific device id and DMA mask overrides can be used - add pds_vdpa.rst into index file - dev_dbg instead of dev_err on most of the adminq commands - rework use of pds_vdpa_cmd_reset() and pds_vdpa_init_hw() for better firmware setup in start-stop-start scenarios - removed unused pds_vdpa_cmd_set_features(), we can rely on vp_modern_set_features() - remove unused hw_qtype and hw_qindex from pds_vdpa_vq_info - reworked debugfs print_feature_bits to also print unknown bits - changed use of PAGE_SIZE to local PDS_PAGE_SIZE to keep with FW layout needs without regard to kernel PAGE_SIZE configuration v2: https://lore.kernel.org/netdev/20230309013046.23523-1-shannon.nel...@amd.com/ - removed PCI driver code - replaced home-grown event listener with notifier - replaced many adminq uses with direct virtio_net config access - reworked irqs to follow virtio layout - removed local_mac_bit logic - replaced uses of devm_ interfaces as suggested in pds_core reviews - updated copyright strings to reflect the new owner Shannon Nelson (11): virtio: allow caller to override device id in vp_modern virtio: allow caller to override device DMA mask in vp_modern pds_vdpa: Add new vDPA driver for AMD/Pensando DSC pds_vdpa: move enum from common to adminq header pds_vdpa: new adminq entries pds_vdpa: get vdpa management info pds_vdpa: virtio bar setup for vdpa pds_vdpa: add vdpa config client commands pds_vdpa: add support for vdpa and vdpamgmt interfaces pds_vdpa: subscribe to the pds_core events pds_vdpa: pds_vdps.rst and Kconfig .../device_drivers/ethernet/amd/pds_vdpa.rst | 85 ++
[PATCH v6 virtio 02/11] virtio: allow caller to override device DMA mask in vp_modern
To add a bit of vendor flexibility with various virtio based devices,
allow the caller to specify a different DMA mask. This adds a dma_mask
field to struct virtio_pci_modern_device. If defined by the driver,
this mask will be used in a call to dma_set_mask_and_coherent() instead
of the traditional DMA_BIT_MASK(64). This allows limiting the DMA space
on vendor devices with address limitations.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/virtio/virtio_pci_modern_dev.c | 3 ++-
include/linux/virtio_pci_modern.h | 3 +++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/drivers/virtio/virtio_pci_modern_dev.c
b/drivers/virtio/virtio_pci_modern_dev.c
index 9b2d6614de67..aad7d9296e77 100644
--- a/drivers/virtio/virtio_pci_modern_dev.c
+++ b/drivers/virtio/virtio_pci_modern_dev.c
@@ -268,7 +268,8 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
return -EINVAL;
}
- err = dma_set_mask_and_coherent(_dev->dev, DMA_BIT_MASK(64));
+ err = dma_set_mask_and_coherent(_dev->dev,
+ mdev->dma_mask ? : DMA_BIT_MASK(64));
if (err)
err = dma_set_mask_and_coherent(_dev->dev,
DMA_BIT_MASK(32));
diff --git a/include/linux/virtio_pci_modern.h
b/include/linux/virtio_pci_modern.h
index e7b1db1dd0bb..067ac1d789bc 100644
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
@@ -41,6 +41,9 @@ struct virtio_pci_modern_device {
/* optional check for vendor virtio device, returns dev_id or -ERRNO */
int (*device_id_check)(struct pci_dev *pdev);
+
+ /* optional mask for devices with limited DMA space */
+ u64 dma_mask;
};
/*
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v6 virtio 01/11] virtio: allow caller to override device id in vp_modern
To add a bit of vendor flexibility with various virtio based devices,
allow the caller to check for a different device id. This adds a function
pointer field to struct virtio_pci_modern_device to specify an override
device id check. If defined by the driver, this function will be called
to check that the PCI device is the vendor's expected device, and will
return the found device id to be stored in mdev->id.device. This allows
vendors with alternative vendor device ids to use this library on their
own device BAR.
Note: A lot of the diff in this is simply indenting the existing code
into an else block.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/virtio/virtio_pci_modern_dev.c | 30 --
include/linux/virtio_pci_modern.h | 3 +++
2 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/drivers/virtio/virtio_pci_modern_dev.c
b/drivers/virtio/virtio_pci_modern_dev.c
index 869cb46bef96..9b2d6614de67 100644
--- a/drivers/virtio/virtio_pci_modern_dev.c
+++ b/drivers/virtio/virtio_pci_modern_dev.c
@@ -218,21 +218,29 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
int err, common, isr, notify, device;
u32 notify_length;
u32 notify_offset;
+ int devid;
check_offsets();
- /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */
- if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)
- return -ENODEV;
-
- if (pci_dev->device < 0x1040) {
- /* Transitional devices: use the PCI subsystem device id as
-* virtio device id, same as legacy driver always did.
-*/
- mdev->id.device = pci_dev->subsystem_device;
+ if (mdev->device_id_check) {
+ devid = mdev->device_id_check(pci_dev);
+ if (devid < 0)
+ return devid;
+ mdev->id.device = devid;
} else {
- /* Modern devices: simply use PCI device id, but start from
0x1040. */
- mdev->id.device = pci_dev->device - 0x1040;
+ /* We only own devices >= 0x1000 and <= 0x107f: leave the rest.
*/
+ if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)
+ return -ENODEV;
+
+ if (pci_dev->device < 0x1040) {
+ /* Transitional devices: use the PCI subsystem device
id as
+* virtio device id, same as legacy driver always did.
+*/
+ mdev->id.device = pci_dev->subsystem_device;
+ } else {
+ /* Modern devices: simply use PCI device id, but start
from 0x1040. */
+ mdev->id.device = pci_dev->device - 0x1040;
+ }
}
mdev->id.vendor = pci_dev->subsystem_vendor;
diff --git a/include/linux/virtio_pci_modern.h
b/include/linux/virtio_pci_modern.h
index c4eeb79b0139..e7b1db1dd0bb 100644
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
@@ -38,6 +38,9 @@ struct virtio_pci_modern_device {
int modern_bars;
struct virtio_device_id id;
+
+ /* optional check for vendor virtio device, returns dev_id or -ERRNO */
+ int (*device_id_check)(struct pci_dev *pdev);
};
/*
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH v2 2/3] vhost: support PACKED when setting-getting vring_base
On 5/9/23 1:46 AM, Stefano Garzarella wrote:
On Mon, Apr 24, 2023 at 03:50:30PM -0700, Shannon Nelson via
Virtualization wrote:
Use the right structs for PACKED or split vqs when setting and
getting the vring base.
Signed-off-by: Shannon Nelson
---
drivers/vhost/vhost.c | 18 +-
drivers/vhost/vhost.h | 8 ++--
2 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index f11bdbe4c2c5..f64efda48f21 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -1633,17 +1633,25 @@ long vhost_vring_ioctl(struct vhost_dev *d,
unsigned int ioctl, void __user *arg
r = -EFAULT;
break;
}
- if (s.num > 0x) {
- r = -EINVAL;
- break;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
+ vq->last_avail_idx = s.num & 0x;
+ vq->last_used_idx = (s.num >> 16) & 0x;
+ } else {
+ if (s.num > 0x) {
+ r = -EINVAL;
+ break;
+ }
+ vq->last_avail_idx = s.num;
}
- vq->last_avail_idx = s.num;
/* Forget the cached index value. */
vq->avail_idx = vq->last_avail_idx;
break;
case VHOST_GET_VRING_BASE:
s.index = idx;
- s.num = vq->last_avail_idx;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED))
+ s.num = (u32)vq->last_avail_idx |
((u32)vq->last_used_idx << 16);
+ else
+ s.num = vq->last_avail_idx;
The changes LGTM, but since we are changing the UAPI, should we
update the documentation of VHOST_SET_VRING_BASE and
VHOST_GET_VRING_BASE in include/uapi/linux/vhost.h?
Correct me if I'm wrong, but I don't think we're changing anything in
the UAPI here, just fixing code to work correctly with what is already
happening.
sln
Thanks,
Stefano
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH v5 virtio 11/11] pds_vdpa: pds_vdps.rst and Kconfig
On 5/14/23 10:07 PM, Jason Wang wrote:
On Thu, May 4, 2023 at 2:13 AM Shannon Nelson wrote:
Add the documentation and Kconfig entry for pds_vdpa driver.
Signed-off-by: Shannon Nelson
---
.../device_drivers/ethernet/amd/pds_vdpa.rst | 85 +++
.../device_drivers/ethernet/index.rst | 1 +
MAINTAINERS | 4 +
drivers/vdpa/Kconfig | 8 ++
4 files changed, 98 insertions(+)
create mode 100644
Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
diff --git a/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
b/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
new file mode 100644
index ..587927d3de92
--- /dev/null
+++ b/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
@@ -0,0 +1,85 @@
+.. SPDX-License-Identifier: GPL-2.0+
+.. note: can be edited and viewed with /usr/bin/formiko-vim
+
+==
+PCI vDPA driver for the AMD/Pensando(R) DSC adapter family
+==
+
+AMD/Pensando vDPA VF Device Driver
+
+Copyright(c) 2023 Advanced Micro Devices, Inc
+
+Overview
+
+
+The ``pds_vdpa`` driver is an auxiliary bus driver that supplies
+a vDPA device for use by the virtio network stack. It is used with
+the Pensando Virtual Function devices that offer vDPA and virtio queue
+services. It depends on the ``pds_core`` driver and hardware for the PF
+and VF PCI handling as well as for device configuration services.
+
+Using the device
+
+
+The ``pds_vdpa`` device is enabled via multiple configuration steps and
+depends on the ``pds_core`` driver to create and enable SR-IOV Virtual
+Function devices. After the VFs are enabled, we enable the vDPA service
+in the ``pds_core`` device to create the auxiliary devices used by pds_vdpa.
+
+Example steps:
+
+.. code-block:: bash
+
+ #!/bin/bash
+
+ modprobe pds_core
+ modprobe vdpa
+ modprobe pds_vdpa
+
+ PF_BDF=`ls /sys/module/pds_core/drivers/pci\:pds_core/*/sriov_numvfs | awk
-F / '{print $7}'`
+
+ # Enable vDPA VF auxiliary device(s) in the PF
+ devlink dev param set pci/$PF_BDF name enable_vnet cmode runtime value true
+
+ # Create a VF for vDPA use
+ echo 1 > /sys/bus/pci/drivers/pds_core/$PF_BDF/sriov_numvfs
+
+ # Find the vDPA services/devices available
+ PDS_VDPA_MGMT=`vdpa mgmtdev show | grep vDPA | head -1 | cut -d: -f1`
+
+ # Create a vDPA device for use in virtio network configurations
+ vdpa dev add name vdpa1 mgmtdev $PDS_VDPA_MGMT mac 00:11:22:33:44:55
+
+ # Set up an ethernet interface on the vdpa device
+ modprobe virtio_vdpa
+
+
+
+Enabling the driver
+===
+
+The driver is enabled via the standard kernel configuration system,
+using the make command::
+
+ make oldconfig/menuconfig/etc.
+
+The driver is located in the menu structure at:
+
+ -> Device Drivers
+-> Network device support (NETDEVICES [=y])
+ -> Ethernet driver support
+-> Pensando devices
+ -> Pensando Ethernet PDS_VDPA Support
+
+Support
+===
+
+For general Linux networking support, please use the netdev mailing
+list, which is monitored by Pensando personnel::
+
+ net...@vger.kernel.org
+
+For more specific support needs, please use the Pensando driver support
+email::
+
+ driv...@pensando.io
diff --git a/Documentation/networking/device_drivers/ethernet/index.rst
b/Documentation/networking/device_drivers/ethernet/index.rst
index 417ca514a4d0..94ecb67c0885 100644
--- a/Documentation/networking/device_drivers/ethernet/index.rst
+++ b/Documentation/networking/device_drivers/ethernet/index.rst
@@ -15,6 +15,7 @@ Contents:
amazon/ena
altera/altera_tse
amd/pds_core
+ amd/pds_vdpa
aquantia/atlantic
chelsio/cxgb
cirrus/cs89x0
diff --git a/MAINTAINERS b/MAINTAINERS
index ebd26b3ca90e..c565b71ce56f 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -22200,6 +22200,10 @@ SNET DPU VIRTIO DATA PATH ACCELERATOR
R: Alvaro Karsz
F: drivers/vdpa/solidrun/
+PDS DSC VIRTIO DATA PATH ACCELERATOR
+R: Shannon Nelson
+F: drivers/vdpa/pds/
+
VIRTIO BALLOON
M: "Michael S. Tsirkin"
M: David Hildenbrand
diff --git a/drivers/vdpa/Kconfig b/drivers/vdpa/Kconfig
index cd6ad92f3f05..2ee1b288691d 100644
--- a/drivers/vdpa/Kconfig
+++ b/drivers/vdpa/Kconfig
@@ -116,4 +116,12 @@ config ALIBABA_ENI_VDPA
This driver includes a HW monitor device that
reads health values from the DPU.
+config PDS_VDPA
+ tristate "vDPA driver for AMD/Pensando DSC devices"
+ depends on PDS_CORE
Need to select VIRTIO_PCI_LIB?
That's a good idea - I'll add that.
sln
Thanks
+ help
+ vDPA network driver for AMD/Pensando's PDS Core devices.
+ With this driver, the VirtIO dataplane can be
+ offloaded to an AMD/Pensando DSC device.
+
endif # VDPA
--
2.17.1
Re: [PATCH v5 virtio 10/11] pds_vdpa: subscribe to the pds_core events
On 5/14/23 10:02 PM, Jason Wang wrote:
On Thu, May 4, 2023 at 2:13 AM Shannon Nelson wrote:
Register for the pds_core's notification events, primarily to
find out when the FW has been reset so we can pass this on
back up the chain.
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/vdpa_dev.c | 68 -
drivers/vdpa/pds/vdpa_dev.h | 1 +
2 files changed, 68 insertions(+), 1 deletion(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index 9970657cdb3d..377eefc2fa1e 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -21,6 +21,61 @@ static struct pds_vdpa_device *vdpa_to_pdsv(struct
vdpa_device *vdpa_dev)
return container_of(vdpa_dev, struct pds_vdpa_device, vdpa_dev);
}
+static int pds_vdpa_notify_handler(struct notifier_block *nb,
+ unsigned long ecode,
+ void *data)
+{
+ struct pds_vdpa_device *pdsv = container_of(nb, struct pds_vdpa_device,
nb);
+ struct device *dev = >vdpa_aux->padev->aux_dev.dev;
+
+ dev_dbg(dev, "%s: event code %lu\n", __func__, ecode);
+
+ /* Give the upper layers a hint that something interesting
+* may have happened. It seems that the only thing this
+* triggers in the virtio-net drivers above us is a check
+* of link status.
+*
+* We don't set the NEEDS_RESET flag for EVENT_RESET
+* because we're likely going through a recovery or
+* fw_update and will be back up and running soon.
+*/
+ if (ecode == PDS_EVENT_RESET || ecode == PDS_EVENT_LINK_CHANGE) {
The code here seems to conflict with the comment above. If we don't
set NEEDS_RESET, there's no need for the config callback?
Yes, that's an out-of-date comment that should be removed. I think we
really just need to pass up the stack the hint that something
interesting may have happened and let the upper layers decide what they
want to do with whatever info they have available.
I'll clean up this comment block.
sln
Thanks
+ if (pdsv->config_cb.callback)
+ pdsv->config_cb.callback(pdsv->config_cb.private);
+ }
+
+ return 0;
+}
+
+static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv)
+{
+ struct device *dev = >vdpa_aux->padev->aux_dev.dev;
+ struct notifier_block *nb = >nb;
+ int err;
+
+ if (!nb->notifier_call) {
+ nb->notifier_call = pds_vdpa_notify_handler;
+ err = pdsc_register_notify(nb);
+ if (err) {
+ nb->notifier_call = NULL;
+ dev_err(dev, "failed to register pds event handler:
%ps\n",
+ ERR_PTR(err));
+ return -EINVAL;
+ }
+ dev_dbg(dev, "pds event handler registered\n");
+ }
+
+ return 0;
+}
+
+static void pds_vdpa_unregister_event_handler(struct pds_vdpa_device *pdsv)
+{
+ if (pdsv->nb.notifier_call) {
+ pdsc_unregister_notify(>nb);
+ pdsv->nb.notifier_call = NULL;
+ }
+}
+
static int pds_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
u64 desc_addr, u64 driver_addr, u64
device_addr)
{
@@ -522,6 +577,12 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
pdsv->vdpa_dev.mdev = _aux->vdpa_mdev;
+ err = pds_vdpa_register_event_handler(pdsv);
+ if (err) {
+ dev_err(dev, "Failed to register for PDS events: %pe\n",
ERR_PTR(err));
+ goto err_unmap;
+ }
+
/* We use the _vdpa_register_device() call rather than the
* vdpa_register_device() to avoid a deadlock because our
* dev_add() is called with the vdpa_dev_lock already set
@@ -530,13 +591,15 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
err = _vdpa_register_device(>vdpa_dev, pdsv->num_vqs);
if (err) {
dev_err(dev, "Failed to register to vDPA bus: %pe\n",
ERR_PTR(err));
- goto err_unmap;
+ goto err_unevent;
}
pds_vdpa_debugfs_add_vdpadev(vdpa_aux);
return 0;
+err_unevent:
+ pds_vdpa_unregister_event_handler(pdsv);
err_unmap:
put_device(>vdpa_dev.dev);
vdpa_aux->pdsv = NULL;
@@ -546,8 +609,11 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
struct vdpa_device *vdpa_dev)
{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
struct pds_vdpa_aux *vdpa_aux;
+ pds_vdpa_unregister_event_handler(pdsv);
+
vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
_vdpa_unregister_device(vdpa_dev);
diff --git
Re: [PATCH] virtio-vdpa: Fix unchecked call to NULL set_vq_affinity
On 5/12/23 6:30 AM, Michael S. Tsirkin wrote:
On Fri, May 12, 2023 at 12:51:21PM +, Dragos Tatulea wrote:
On Thu, 2023-05-04 at 14:51 -0400, Michael S. Tsirkin wrote:
On Thu, May 04, 2023 at 01:08:54PM -0400, Feng Liu wrote:
On 2023-05-04 a.m.9:50, Dragos Tatulea wrote:
External email: Use caution opening links or attachments
The referenced patch calls set_vq_affinity without checking if the op is
valid. This patch adds the check.
Fixes: 3dad56823b53 ("virtio-vdpa: Support interrupt affinity spreading
mechanism")
Reviewed-by: Gal Pressman
Signed-off-by: Dragos Tatulea
---
drivers/virtio/virtio_vdpa.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
index eb6aee8c06b2..989e2d7184ce 100644
--- a/drivers/virtio/virtio_vdpa.c
+++ b/drivers/virtio/virtio_vdpa.c
@@ -385,7 +385,9 @@ static int virtio_vdpa_find_vqs(struct virtio_device
*vdev, unsigned int nvqs,
err = PTR_ERR(vqs[i]);
goto err_setup_vq;
}
- ops->set_vq_affinity(vdpa, i, [i]);
+
+ if (ops->set_vq_affinity)
+ ops->set_vq_affinity(vdpa, i, [i]);
if ops->set_vq_affinity is NULL, should give an error code to err, and
return err
Given we ignore return code, hardly seems like a critical thing to do.
Is it really important? affinity is an optimization isn't it?
set_vq_affinity is optional so it's not an error if the op is not implemented.
Is there anything else that needs to be done for this fix?
Thanks,
Dragos
no, it's queued already.
Are these queued into a repo that is accessible? I haven't seen
activity in the vhost.git where I would have expected it. After
stumbling over and debugging this same problem, I was happy to see it
fixed, and I'd like to pull from a repo that has the current updates.
Thanks,
sln
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v5 virtio 09/11] pds_vdpa: add support for vdpa and vdpamgmt interfaces
This is the vDPA device support, where we advertise that we can
support the virtio queues and deal with the configuration work
through the pds_core's adminq.
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/aux_drv.c | 15 +
drivers/vdpa/pds/aux_drv.h | 1 +
drivers/vdpa/pds/debugfs.c | 261 ++
drivers/vdpa/pds/debugfs.h | 5 +
drivers/vdpa/pds/vdpa_dev.c | 532 +++-
5 files changed, 813 insertions(+), 1 deletion(-)
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
index 0c4a135b1484..186e9ee22eb1 100644
--- a/drivers/vdpa/pds/aux_drv.c
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -63,8 +63,21 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
goto err_free_mgmt_info;
}
+ /* Let vdpa know that we can provide devices */
+ err = vdpa_mgmtdev_register(_aux->vdpa_mdev);
+ if (err) {
+ dev_err(dev, "%s: Failed to initialize vdpa_mgmt interface:
%pe\n",
+ __func__, ERR_PTR(err));
+ goto err_free_virtio;
+ }
+
+ pds_vdpa_debugfs_add_pcidev(vdpa_aux);
+ pds_vdpa_debugfs_add_ident(vdpa_aux);
+
return 0;
+err_free_virtio:
+ vp_modern_remove(_aux->vd_mdev);
err_free_mgmt_info:
pci_free_irq_vectors(padev->vf_pdev);
err_free_mem:
@@ -79,9 +92,11 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
struct device *dev = _dev->dev;
+ vdpa_mgmtdev_unregister(_aux->vdpa_mdev);
vp_modern_remove(_aux->vd_mdev);
pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
+ pds_vdpa_debugfs_del_vdpadev(vdpa_aux);
kfree(vdpa_aux);
auxiliary_set_drvdata(aux_dev, NULL);
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
index 99e0ff340bfa..26b75344156e 100644
--- a/drivers/vdpa/pds/aux_drv.h
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -13,6 +13,7 @@ struct pds_vdpa_aux {
struct pds_auxiliary_dev *padev;
struct vdpa_mgmt_dev vdpa_mdev;
+ struct pds_vdpa_device *pdsv;
struct pds_vdpa_ident ident;
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
index d91dceb07380..0ecd0e2ec6b9 100644
--- a/drivers/vdpa/pds/debugfs.c
+++ b/drivers/vdpa/pds/debugfs.c
@@ -10,6 +10,7 @@
#include
#include "aux_drv.h"
+#include "vdpa_dev.h"
#include "debugfs.h"
static struct dentry *dbfs_dir;
@@ -24,3 +25,263 @@ void pds_vdpa_debugfs_destroy(void)
debugfs_remove_recursive(dbfs_dir);
dbfs_dir = NULL;
}
+
+#define PRINT_SBIT_NAME(__seq, __f, __name) \
+ do {\
+ if ((__f) & (__name)) \
+ seq_printf(__seq, " %s",
[PATCH v5 virtio 10/11] pds_vdpa: subscribe to the pds_core events
Register for the pds_core's notification events, primarily to
find out when the FW has been reset so we can pass this on
back up the chain.
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/vdpa_dev.c | 68 -
drivers/vdpa/pds/vdpa_dev.h | 1 +
2 files changed, 68 insertions(+), 1 deletion(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index 9970657cdb3d..377eefc2fa1e 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -21,6 +21,61 @@ static struct pds_vdpa_device *vdpa_to_pdsv(struct
vdpa_device *vdpa_dev)
return container_of(vdpa_dev, struct pds_vdpa_device, vdpa_dev);
}
+static int pds_vdpa_notify_handler(struct notifier_block *nb,
+ unsigned long ecode,
+ void *data)
+{
+ struct pds_vdpa_device *pdsv = container_of(nb, struct pds_vdpa_device,
nb);
+ struct device *dev = >vdpa_aux->padev->aux_dev.dev;
+
+ dev_dbg(dev, "%s: event code %lu\n", __func__, ecode);
+
+ /* Give the upper layers a hint that something interesting
+* may have happened. It seems that the only thing this
+* triggers in the virtio-net drivers above us is a check
+* of link status.
+*
+* We don't set the NEEDS_RESET flag for EVENT_RESET
+* because we're likely going through a recovery or
+* fw_update and will be back up and running soon.
+*/
+ if (ecode == PDS_EVENT_RESET || ecode == PDS_EVENT_LINK_CHANGE) {
+ if (pdsv->config_cb.callback)
+ pdsv->config_cb.callback(pdsv->config_cb.private);
+ }
+
+ return 0;
+}
+
+static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv)
+{
+ struct device *dev = >vdpa_aux->padev->aux_dev.dev;
+ struct notifier_block *nb = >nb;
+ int err;
+
+ if (!nb->notifier_call) {
+ nb->notifier_call = pds_vdpa_notify_handler;
+ err = pdsc_register_notify(nb);
+ if (err) {
+ nb->notifier_call = NULL;
+ dev_err(dev, "failed to register pds event handler:
%ps\n",
+ ERR_PTR(err));
+ return -EINVAL;
+ }
+ dev_dbg(dev, "pds event handler registered\n");
+ }
+
+ return 0;
+}
+
+static void pds_vdpa_unregister_event_handler(struct pds_vdpa_device *pdsv)
+{
+ if (pdsv->nb.notifier_call) {
+ pdsc_unregister_notify(>nb);
+ pdsv->nb.notifier_call = NULL;
+ }
+}
+
static int pds_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
u64 desc_addr, u64 driver_addr, u64
device_addr)
{
@@ -522,6 +577,12 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
pdsv->vdpa_dev.mdev = _aux->vdpa_mdev;
+ err = pds_vdpa_register_event_handler(pdsv);
+ if (err) {
+ dev_err(dev, "Failed to register for PDS events: %pe\n",
ERR_PTR(err));
+ goto err_unmap;
+ }
+
/* We use the _vdpa_register_device() call rather than the
* vdpa_register_device() to avoid a deadlock because our
* dev_add() is called with the vdpa_dev_lock already set
@@ -530,13 +591,15 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
err = _vdpa_register_device(>vdpa_dev, pdsv->num_vqs);
if (err) {
dev_err(dev, "Failed to register to vDPA bus: %pe\n",
ERR_PTR(err));
- goto err_unmap;
+ goto err_unevent;
}
pds_vdpa_debugfs_add_vdpadev(vdpa_aux);
return 0;
+err_unevent:
+ pds_vdpa_unregister_event_handler(pdsv);
err_unmap:
put_device(>vdpa_dev.dev);
vdpa_aux->pdsv = NULL;
@@ -546,8 +609,11 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
struct vdpa_device *vdpa_dev)
{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
struct pds_vdpa_aux *vdpa_aux;
+ pds_vdpa_unregister_event_handler(pdsv);
+
vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
_vdpa_unregister_device(vdpa_dev);
diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h
index a21596f438c1..1650a2b08845 100644
--- a/drivers/vdpa/pds/vdpa_dev.h
+++ b/drivers/vdpa/pds/vdpa_dev.h
@@ -40,6 +40,7 @@ struct pds_vdpa_device {
u8 vdpa_index; /* rsvd for future subdevice use */
u8 num_vqs; /* num vqs in use */
struct vdpa_callback config_cb;
+ struct notifier_block nb;
};
int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux);
--
2.17.1
___
Virtualization mailing list
[PATCH v5 virtio 08/11] pds_vdpa: add vdpa config client commands
These are the adminq commands that will be needed for
setting up and using the vDPA device. There are a number
of commands defined in the FW's API, but by making use of
the FW's virtio BAR we only need a few of these commands
for vDPA support.
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/Makefile | 1 +
drivers/vdpa/pds/cmds.c | 207
drivers/vdpa/pds/cmds.h | 20
drivers/vdpa/pds/vdpa_dev.h | 33 +-
4 files changed, 260 insertions(+), 1 deletion(-)
create mode 100644 drivers/vdpa/pds/cmds.c
create mode 100644 drivers/vdpa/pds/cmds.h
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
index 13b50394ec64..2e22418e3ab3 100644
--- a/drivers/vdpa/pds/Makefile
+++ b/drivers/vdpa/pds/Makefile
@@ -4,6 +4,7 @@
obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
pds_vdpa-y := aux_drv.o \
+ cmds.o \
vdpa_dev.o
pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/vdpa/pds/cmds.c b/drivers/vdpa/pds/cmds.c
new file mode 100644
index ..405711a0a0f8
--- /dev/null
+++ b/drivers/vdpa/pds/cmds.c
@@ -0,0 +1,207 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include "vdpa_dev.h"
+#include "aux_drv.h"
+#include "cmds.h"
+
+int pds_vdpa_init_hw(struct pds_vdpa_device *pdsv)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_init.opcode = PDS_VDPA_CMD_INIT,
+ .vdpa_init.vdpa_index = pdsv->vdpa_index,
+ .vdpa_init.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ /* Initialize the vdpa/virtio device */
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa_init),
+ , 0);
+ if (err)
+ dev_dbg(dev, "Failed to init hw, status %d: %pe\n",
+ comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_reset(struct pds_vdpa_device *pdsv)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa.opcode = PDS_VDPA_CMD_RESET,
+ .vdpa.vdpa_index = pdsv->vdpa_index,
+ .vdpa.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa), , 0);
+ if (err)
+ dev_dbg(dev, "Failed to reset hw, status %d: %pe\n",
+ comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_set_mac(struct pds_vdpa_device *pdsv, u8 *mac)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR,
+ .vdpa_setattr.vdpa_index = pdsv->vdpa_index,
+ .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ .vdpa_setattr.attr = PDS_VDPA_ATTR_MAC,
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ ether_addr_copy(cmd.vdpa_setattr.mac, mac);
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa_setattr),
+ , 0);
+ if (err)
+ dev_dbg(dev, "Failed to set mac address %pM, status %d: %pe\n",
+ mac, comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_set_max_vq_pairs(struct pds_vdpa_device *pdsv, u16 max_vqp)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR,
+ .vdpa_setattr.vdpa_index = pdsv->vdpa_index,
+ .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ .vdpa_setattr.attr = PDS_VDPA_ATTR_MAX_VQ_PAIRS,
+ .vdpa_setattr.max_vq_pairs = cpu_to_le16(max_vqp),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa_setattr),
+ , 0);
+ if (err)
+ dev_dbg(dev, "Failed to set max vq pairs %u, status %d: %pe\n",
+ max_vqp, comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_init_vq(struct pds_vdpa_device *pdsv, u16 qid,
+struct pds_vdpa_vq_info *vq_info)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_vq_init.opcode =
[PATCH v5 virtio 05/11] pds_vdpa: new adminq entries
Add new adminq definitions in support for vDPA operations.
Signed-off-by: Shannon Nelson
---
include/linux/pds/pds_adminq.h | 266 +
1 file changed, 266 insertions(+)
diff --git a/include/linux/pds/pds_adminq.h b/include/linux/pds/pds_adminq.h
index 61b0a8634e1a..c66ead725434 100644
--- a/include/linux/pds/pds_adminq.h
+++ b/include/linux/pds/pds_adminq.h
@@ -605,6 +605,257 @@ struct pds_core_q_init_comp {
u8 color;
};
+/*
+ * enum pds_vdpa_cmd_opcode - vDPA Device commands
+ */
+enum pds_vdpa_cmd_opcode {
+ PDS_VDPA_CMD_INIT = 48,
+ PDS_VDPA_CMD_IDENT = 49,
+ PDS_VDPA_CMD_RESET = 51,
+ PDS_VDPA_CMD_VQ_RESET = 52,
+ PDS_VDPA_CMD_VQ_INIT= 53,
+ PDS_VDPA_CMD_STATUS_UPDATE = 54,
+ PDS_VDPA_CMD_SET_FEATURES = 55,
+ PDS_VDPA_CMD_SET_ATTR = 56,
+ PDS_VDPA_CMD_VQ_SET_STATE = 57,
+ PDS_VDPA_CMD_VQ_GET_STATE = 58,
+};
+
+/**
+ * struct pds_vdpa_cmd - generic command
+ * @opcode:Opcode
+ * @vdpa_index:Index for vdpa subdevice
+ * @vf_id: VF id
+ */
+struct pds_vdpa_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+};
+
+/**
+ * struct pds_vdpa_init_cmd - INIT command
+ * @opcode:Opcode PDS_VDPA_CMD_INIT
+ * @vdpa_index: Index for vdpa subdevice
+ * @vf_id: VF id
+ */
+struct pds_vdpa_init_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+};
+
+/**
+ * struct pds_vdpa_ident - vDPA identification data
+ * @hw_features: vDPA features supported by device
+ * @max_vqs: max queues available (2 queues for a single queuepair)
+ * @max_qlen: log(2) of maximum number of descriptors
+ * @min_qlen: log(2) of minimum number of descriptors
+ *
+ * This struct is used in a DMA block that is set up for the PDS_VDPA_CMD_IDENT
+ * transaction. Set up the DMA block and send the address in the IDENT cmd
+ * data, the DSC will write the ident information, then we can remove the DMA
+ * block after reading the answer. If the completion status is 0, then there
+ * is valid information, else there was an error and the data should be
invalid.
+ */
+struct pds_vdpa_ident {
+ __le64 hw_features;
+ __le16 max_vqs;
+ __le16 max_qlen;
+ __le16 min_qlen;
+};
+
+/**
+ * struct pds_vdpa_ident_cmd - IDENT command
+ * @opcode:Opcode PDS_VDPA_CMD_IDENT
+ * @rsvd: Word boundary padding
+ * @vf_id: VF id
+ * @len: length of ident info DMA space
+ * @ident_pa: address for DMA of ident info (struct pds_vdpa_ident)
+ * only used for this transaction, then forgotten by DSC
+ */
+struct pds_vdpa_ident_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 vf_id;
+ __le32 len;
+ __le64 ident_pa;
+};
+
+/**
+ * struct pds_vdpa_status_cmd - STATUS_UPDATE command
+ * @opcode:Opcode PDS_VDPA_CMD_STATUS_UPDATE
+ * @vdpa_index: Index for vdpa subdevice
+ * @vf_id: VF id
+ * @status:new status bits
+ */
+struct pds_vdpa_status_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+ u8 status;
+};
+
+/**
+ * enum pds_vdpa_attr - List of VDPA device attributes
+ * @PDS_VDPA_ATTR_MAC: MAC address
+ * @PDS_VDPA_ATTR_MAX_VQ_PAIRS: Max virtqueue pairs
+ */
+enum pds_vdpa_attr {
+ PDS_VDPA_ATTR_MAC = 1,
+ PDS_VDPA_ATTR_MAX_VQ_PAIRS = 2,
+};
+
+/**
+ * struct pds_vdpa_setattr_cmd - SET_ATTR command
+ * @opcode:Opcode PDS_VDPA_CMD_SET_ATTR
+ * @vdpa_index:Index for vdpa subdevice
+ * @vf_id: VF id
+ * @attr: attribute to be changed (enum pds_vdpa_attr)
+ * @pad: Word boundary padding
+ * @mac: new mac address to be assigned as vdpa device address
+ * @max_vq_pairs: new limit of virtqueue pairs
+ */
+struct pds_vdpa_setattr_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+ u8 attr;
+ u8 pad[3];
+ union {
+ u8 mac[6];
+ __le16 max_vq_pairs;
+ } __packed;
+};
+
+/**
+ * struct pds_vdpa_vq_init_cmd - queue init command
+ * @opcode: Opcode PDS_VDPA_CMD_VQ_INIT
+ * @vdpa_index:Index for vdpa subdevice
+ * @vf_id: VF id
+ * @qid: Queue id (bit0 clear = rx, bit0 set = tx, qid=N is ctrlq)
+ * @len: log(2) of max descriptor count
+ * @desc_addr: DMA address of descriptor area
+ * @avail_addr:DMA address of available descriptors (aka driver area)
+ * @used_addr: DMA address of used descriptors (aka device area)
+ * @intr_index:interrupt index
+ */
+struct pds_vdpa_vq_init_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+ __le16 qid;
+ __le16 len;
+ __le64 desc_addr;
+ __le64 avail_addr;
+ __le64 used_addr;
+ __le16 intr_index;
[PATCH v5 virtio 03/11] pds_vdpa: Add new vDPA driver for AMD/Pensando DSC
This is the initial auxiliary driver framework for a new vDPA
device driver, an auxiliary_bus client of the pds_core driver.
The pds_core driver supplies the PCI services for the VF device
and for accessing the adminq in the PF device.
This patch adds the very basics of registering for the auxiliary
device and setting up debugfs entries.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/Makefile | 1 +
drivers/vdpa/pds/Makefile | 8
drivers/vdpa/pds/aux_drv.c | 83 ++
drivers/vdpa/pds/aux_drv.h | 15 ++
drivers/vdpa/pds/debugfs.c | 25 ++
drivers/vdpa/pds/debugfs.h | 12 +
include/linux/pds/pds_common.h | 2 +
7 files changed, 146 insertions(+)
create mode 100644 drivers/vdpa/pds/Makefile
create mode 100644 drivers/vdpa/pds/aux_drv.c
create mode 100644 drivers/vdpa/pds/aux_drv.h
create mode 100644 drivers/vdpa/pds/debugfs.c
create mode 100644 drivers/vdpa/pds/debugfs.h
diff --git a/drivers/vdpa/Makefile b/drivers/vdpa/Makefile
index 59396ff2a318..8f53c6f3cca7 100644
--- a/drivers/vdpa/Makefile
+++ b/drivers/vdpa/Makefile
@@ -7,3 +7,4 @@ obj-$(CONFIG_MLX5_VDPA) += mlx5/
obj-$(CONFIG_VP_VDPA)+= virtio_pci/
obj-$(CONFIG_ALIBABA_ENI_VDPA) += alibaba/
obj-$(CONFIG_SNET_VDPA) += solidrun/
+obj-$(CONFIG_PDS_VDPA) += pds/
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
new file mode 100644
index ..a9cd2f450ae1
--- /dev/null
+++ b/drivers/vdpa/pds/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# Copyright(c) 2023 Advanced Micro Devices, Inc
+
+obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
+
+pds_vdpa-y := aux_drv.o
+
+pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
new file mode 100644
index ..e4a0ad61ea22
--- /dev/null
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include "aux_drv.h"
+#include "debugfs.h"
+
+static const struct auxiliary_device_id pds_vdpa_id_table[] = {
+ { .name = PDS_VDPA_DEV_NAME, },
+ {},
+};
+
+static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
+ const struct auxiliary_device_id *id)
+
+{
+ struct pds_auxiliary_dev *padev =
+ container_of(aux_dev, struct pds_auxiliary_dev, aux_dev);
+ struct pds_vdpa_aux *vdpa_aux;
+
+ vdpa_aux = kzalloc(sizeof(*vdpa_aux), GFP_KERNEL);
+ if (!vdpa_aux)
+ return -ENOMEM;
+
+ vdpa_aux->padev = padev;
+ auxiliary_set_drvdata(aux_dev, vdpa_aux);
+
+ return 0;
+}
+
+static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
+{
+ struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
+ struct device *dev = _dev->dev;
+
+ kfree(vdpa_aux);
+ auxiliary_set_drvdata(aux_dev, NULL);
+
+ dev_info(dev, "Removed\n");
+}
+
+static struct auxiliary_driver pds_vdpa_driver = {
+ .name = PDS_DEV_TYPE_VDPA_STR,
+ .probe = pds_vdpa_probe,
+ .remove = pds_vdpa_remove,
+ .id_table = pds_vdpa_id_table,
+};
+
+static void __exit pds_vdpa_cleanup(void)
+{
+ auxiliary_driver_unregister(_vdpa_driver);
+
+ pds_vdpa_debugfs_destroy();
+}
+module_exit(pds_vdpa_cleanup);
+
+static int __init pds_vdpa_init(void)
+{
+ int err;
+
+ pds_vdpa_debugfs_create();
+
+ err = auxiliary_driver_register(_vdpa_driver);
+ if (err) {
+ pr_err("%s: aux driver register failed: %pe\n",
+ PDS_VDPA_DRV_NAME, ERR_PTR(err));
+ pds_vdpa_debugfs_destroy();
+ }
+
+ return err;
+}
+module_init(pds_vdpa_init);
+
+MODULE_DESCRIPTION(PDS_VDPA_DRV_DESCRIPTION);
+MODULE_AUTHOR("Advanced Micro Devices, Inc");
+MODULE_LICENSE("GPL");
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
new file mode 100644
index ..f1e99359424e
--- /dev/null
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#ifndef _AUX_DRV_H_
+#define _AUX_DRV_H_
+
+#define PDS_VDPA_DRV_DESCRIPTION"AMD/Pensando vDPA VF Device Driver"
+#define PDS_VDPA_DRV_NAME KBUILD_MODNAME
+
+struct pds_vdpa_aux {
+ struct pds_auxiliary_dev *padev;
+
+ struct dentry *dentry;
+};
+#endif /* _AUX_DRV_H_ */
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
new file mode 100644
index ..5be22fb7a76a
--- /dev/null
+++ b/drivers/vdpa/pds/debugfs.c
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+
+#include
+#include
+#include
+#include
+
+#include "aux_drv.h"
+#include "debugfs.h"
+
+static struct dentry *dbfs_dir;
+
+void
[PATCH v5 virtio 11/11] pds_vdpa: pds_vdps.rst and Kconfig
Add the documentation and Kconfig entry for pds_vdpa driver.
Signed-off-by: Shannon Nelson
---
.../device_drivers/ethernet/amd/pds_vdpa.rst | 85 +++
.../device_drivers/ethernet/index.rst | 1 +
MAINTAINERS | 4 +
drivers/vdpa/Kconfig | 8 ++
4 files changed, 98 insertions(+)
create mode 100644
Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
diff --git a/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
b/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
new file mode 100644
index ..587927d3de92
--- /dev/null
+++ b/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
@@ -0,0 +1,85 @@
+.. SPDX-License-Identifier: GPL-2.0+
+.. note: can be edited and viewed with /usr/bin/formiko-vim
+
+==
+PCI vDPA driver for the AMD/Pensando(R) DSC adapter family
+==
+
+AMD/Pensando vDPA VF Device Driver
+
+Copyright(c) 2023 Advanced Micro Devices, Inc
+
+Overview
+
+
+The ``pds_vdpa`` driver is an auxiliary bus driver that supplies
+a vDPA device for use by the virtio network stack. It is used with
+the Pensando Virtual Function devices that offer vDPA and virtio queue
+services. It depends on the ``pds_core`` driver and hardware for the PF
+and VF PCI handling as well as for device configuration services.
+
+Using the device
+
+
+The ``pds_vdpa`` device is enabled via multiple configuration steps and
+depends on the ``pds_core`` driver to create and enable SR-IOV Virtual
+Function devices. After the VFs are enabled, we enable the vDPA service
+in the ``pds_core`` device to create the auxiliary devices used by pds_vdpa.
+
+Example steps:
+
+.. code-block:: bash
+
+ #!/bin/bash
+
+ modprobe pds_core
+ modprobe vdpa
+ modprobe pds_vdpa
+
+ PF_BDF=`ls /sys/module/pds_core/drivers/pci\:pds_core/*/sriov_numvfs | awk
-F / '{print $7}'`
+
+ # Enable vDPA VF auxiliary device(s) in the PF
+ devlink dev param set pci/$PF_BDF name enable_vnet cmode runtime value true
+
+ # Create a VF for vDPA use
+ echo 1 > /sys/bus/pci/drivers/pds_core/$PF_BDF/sriov_numvfs
+
+ # Find the vDPA services/devices available
+ PDS_VDPA_MGMT=`vdpa mgmtdev show | grep vDPA | head -1 | cut -d: -f1`
+
+ # Create a vDPA device for use in virtio network configurations
+ vdpa dev add name vdpa1 mgmtdev $PDS_VDPA_MGMT mac 00:11:22:33:44:55
+
+ # Set up an ethernet interface on the vdpa device
+ modprobe virtio_vdpa
+
+
+
+Enabling the driver
+===
+
+The driver is enabled via the standard kernel configuration system,
+using the make command::
+
+ make oldconfig/menuconfig/etc.
+
+The driver is located in the menu structure at:
+
+ -> Device Drivers
+-> Network device support (NETDEVICES [=y])
+ -> Ethernet driver support
+-> Pensando devices
+ -> Pensando Ethernet PDS_VDPA Support
+
+Support
+===
+
+For general Linux networking support, please use the netdev mailing
+list, which is monitored by Pensando personnel::
+
+ net...@vger.kernel.org
+
+For more specific support needs, please use the Pensando driver support
+email::
+
+ driv...@pensando.io
diff --git a/Documentation/networking/device_drivers/ethernet/index.rst
b/Documentation/networking/device_drivers/ethernet/index.rst
index 417ca514a4d0..94ecb67c0885 100644
--- a/Documentation/networking/device_drivers/ethernet/index.rst
+++ b/Documentation/networking/device_drivers/ethernet/index.rst
@@ -15,6 +15,7 @@ Contents:
amazon/ena
altera/altera_tse
amd/pds_core
+ amd/pds_vdpa
aquantia/atlantic
chelsio/cxgb
cirrus/cs89x0
diff --git a/MAINTAINERS b/MAINTAINERS
index ebd26b3ca90e..c565b71ce56f 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -22200,6 +22200,10 @@ SNET DPU VIRTIO DATA PATH ACCELERATOR
R: Alvaro Karsz
F: drivers/vdpa/solidrun/
+PDS DSC VIRTIO DATA PATH ACCELERATOR
+R: Shannon Nelson
+F: drivers/vdpa/pds/
+
VIRTIO BALLOON
M: "Michael S. Tsirkin"
M: David Hildenbrand
diff --git a/drivers/vdpa/Kconfig b/drivers/vdpa/Kconfig
index cd6ad92f3f05..2ee1b288691d 100644
--- a/drivers/vdpa/Kconfig
+++ b/drivers/vdpa/Kconfig
@@ -116,4 +116,12 @@ config ALIBABA_ENI_VDPA
This driver includes a HW monitor device that
reads health values from the DPU.
+config PDS_VDPA
+ tristate "vDPA driver for AMD/Pensando DSC devices"
+ depends on PDS_CORE
+ help
+ vDPA network driver for AMD/Pensando's PDS Core devices.
+ With this driver, the VirtIO dataplane can be
+ offloaded to an AMD/Pensando DSC device.
+
endif # VDPA
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v5 virtio 06/11] pds_vdpa: get vdpa management info
Find the vDPA management information from the DSC in order to
advertise it to the vdpa subsystem.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/pds/Makefile | 3 +-
drivers/vdpa/pds/aux_drv.c | 17 ++
drivers/vdpa/pds/aux_drv.h | 7 +++
drivers/vdpa/pds/debugfs.c | 1 +
drivers/vdpa/pds/vdpa_dev.c | 108
drivers/vdpa/pds/vdpa_dev.h | 15 +
6 files changed, 150 insertions(+), 1 deletion(-)
create mode 100644 drivers/vdpa/pds/vdpa_dev.c
create mode 100644 drivers/vdpa/pds/vdpa_dev.h
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
index a9cd2f450ae1..13b50394ec64 100644
--- a/drivers/vdpa/pds/Makefile
+++ b/drivers/vdpa/pds/Makefile
@@ -3,6 +3,7 @@
obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
-pds_vdpa-y := aux_drv.o
+pds_vdpa-y := aux_drv.o \
+ vdpa_dev.o
pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
index e4a0ad61ea22..aa748cf55d2b 100644
--- a/drivers/vdpa/pds/aux_drv.c
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -3,6 +3,7 @@
#include
#include
+#include
#include
#include
@@ -11,6 +12,7 @@
#include "aux_drv.h"
#include "debugfs.h"
+#include "vdpa_dev.h"
static const struct auxiliary_device_id pds_vdpa_id_table[] = {
{ .name = PDS_VDPA_DEV_NAME, },
@@ -24,15 +26,28 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
struct pds_auxiliary_dev *padev =
container_of(aux_dev, struct pds_auxiliary_dev, aux_dev);
struct pds_vdpa_aux *vdpa_aux;
+ int err;
vdpa_aux = kzalloc(sizeof(*vdpa_aux), GFP_KERNEL);
if (!vdpa_aux)
return -ENOMEM;
vdpa_aux->padev = padev;
+ vdpa_aux->vf_id = pci_iov_vf_id(padev->vf_pdev);
auxiliary_set_drvdata(aux_dev, vdpa_aux);
+ /* Get device ident info and set up the vdpa_mgmt_dev */
+ err = pds_vdpa_get_mgmt_info(vdpa_aux);
+ if (err)
+ goto err_free_mem;
+
return 0;
+
+err_free_mem:
+ kfree(vdpa_aux);
+ auxiliary_set_drvdata(aux_dev, NULL);
+
+ return err;
}
static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
@@ -40,6 +55,8 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
struct device *dev = _dev->dev;
+ pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
+
kfree(vdpa_aux);
auxiliary_set_drvdata(aux_dev, NULL);
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
index f1e99359424e..dcec782e79eb 100644
--- a/drivers/vdpa/pds/aux_drv.h
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -10,6 +10,13 @@
struct pds_vdpa_aux {
struct pds_auxiliary_dev *padev;
+ struct vdpa_mgmt_dev vdpa_mdev;
+
+ struct pds_vdpa_ident ident;
+
+ int vf_id;
struct dentry *dentry;
+
+ int nintrs;
};
#endif /* _AUX_DRV_H_ */
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
index 5be22fb7a76a..d91dceb07380 100644
--- a/drivers/vdpa/pds/debugfs.c
+++ b/drivers/vdpa/pds/debugfs.c
@@ -2,6 +2,7 @@
/* Copyright(c) 2023 Advanced Micro Devices, Inc */
#include
+#include
#include
#include
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
new file mode 100644
index ..0f0f0ab8b811
--- /dev/null
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include "vdpa_dev.h"
+#include "aux_drv.h"
+
+static struct virtio_device_id pds_vdpa_id_table[] = {
+ {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
+ {0},
+};
+
+static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
+ const struct vdpa_dev_set_config *add_config)
+{
+ return -EOPNOTSUPP;
+}
+
+static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
+struct vdpa_device *vdpa_dev)
+{
+}
+
+static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = {
+ .dev_add = pds_vdpa_dev_add,
+ .dev_del = pds_vdpa_dev_del
+};
+
+int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
+{
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_ident.opcode = PDS_VDPA_CMD_IDENT,
+ .vdpa_ident.vf_id = cpu_to_le16(vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ struct vdpa_mgmt_dev *mgmt;
+ struct pci_dev *pf_pdev;
+ struct device *pf_dev;
+ struct pci_dev *pdev;
+ dma_addr_t ident_pa;
+ struct device *dev;
+ u16 dev_intrs;
+ u16 max_vqs;
+ int err;
+
+ dev = _aux->padev->aux_dev.dev;
+ pdev = vdpa_aux->padev->vf_pdev;
+ mgmt = _aux->vdpa_mdev;
+
+ /* Get resource info through the PF's adminq. It
[PATCH v5 virtio 01/11] virtio: allow caller to override device id in vp_modern
To add a bit of vendor flexibility with various virtio based devices,
allow the caller to check for a different device id. This adds a function
pointer field to struct virtio_pci_modern_device to specify an override
device id check. If defined by the driver, this function will be called
to check that the PCI device is the vendor's expected device, and will
return the found device id to be stored in mdev->id.device. This allows
vendors with alternative vendor device ids to use this library on their
own device BAR.
Note: A lot of the diff in this is simply indenting the existing code
into an else block.
Signed-off-by: Shannon Nelson
---
drivers/virtio/virtio_pci_modern_dev.c | 30 --
include/linux/virtio_pci_modern.h | 3 +++
2 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/drivers/virtio/virtio_pci_modern_dev.c
b/drivers/virtio/virtio_pci_modern_dev.c
index 869cb46bef96..9b2d6614de67 100644
--- a/drivers/virtio/virtio_pci_modern_dev.c
+++ b/drivers/virtio/virtio_pci_modern_dev.c
@@ -218,21 +218,29 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
int err, common, isr, notify, device;
u32 notify_length;
u32 notify_offset;
+ int devid;
check_offsets();
- /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */
- if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)
- return -ENODEV;
-
- if (pci_dev->device < 0x1040) {
- /* Transitional devices: use the PCI subsystem device id as
-* virtio device id, same as legacy driver always did.
-*/
- mdev->id.device = pci_dev->subsystem_device;
+ if (mdev->device_id_check) {
+ devid = mdev->device_id_check(pci_dev);
+ if (devid < 0)
+ return devid;
+ mdev->id.device = devid;
} else {
- /* Modern devices: simply use PCI device id, but start from
0x1040. */
- mdev->id.device = pci_dev->device - 0x1040;
+ /* We only own devices >= 0x1000 and <= 0x107f: leave the rest.
*/
+ if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)
+ return -ENODEV;
+
+ if (pci_dev->device < 0x1040) {
+ /* Transitional devices: use the PCI subsystem device
id as
+* virtio device id, same as legacy driver always did.
+*/
+ mdev->id.device = pci_dev->subsystem_device;
+ } else {
+ /* Modern devices: simply use PCI device id, but start
from 0x1040. */
+ mdev->id.device = pci_dev->device - 0x1040;
+ }
}
mdev->id.vendor = pci_dev->subsystem_vendor;
diff --git a/include/linux/virtio_pci_modern.h
b/include/linux/virtio_pci_modern.h
index c4eeb79b0139..e7b1db1dd0bb 100644
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
@@ -38,6 +38,9 @@ struct virtio_pci_modern_device {
int modern_bars;
struct virtio_device_id id;
+
+ /* optional check for vendor virtio device, returns dev_id or -ERRNO */
+ int (*device_id_check)(struct pci_dev *pdev);
};
/*
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v5 virtio 04/11] pds_vdpa: move enum from common to adminq header
The pds_core_logical_qtype enum and IFNAMSIZ are not needed
in the common PDS header, only needed when working with the
adminq, so move them to the adminq header.
Note: This patch might conflict with pds_vfio patches that are
in review, depending on which patchset gets pulled first.
Signed-off-by: Shannon Nelson
---
include/linux/pds/pds_adminq.h | 21 +
include/linux/pds/pds_common.h | 21 -
2 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/include/linux/pds/pds_adminq.h b/include/linux/pds/pds_adminq.h
index 98a60ce87b92..61b0a8634e1a 100644
--- a/include/linux/pds/pds_adminq.h
+++ b/include/linux/pds/pds_adminq.h
@@ -222,6 +222,27 @@ enum pds_core_lif_type {
PDS_CORE_LIF_TYPE_DEFAULT = 0,
};
+#define PDS_CORE_IFNAMSIZ 16
+
+/**
+ * enum pds_core_logical_qtype - Logical Queue Types
+ * @PDS_CORE_QTYPE_ADMINQ:Administrative Queue
+ * @PDS_CORE_QTYPE_NOTIFYQ: Notify Queue
+ * @PDS_CORE_QTYPE_RXQ: Receive Queue
+ * @PDS_CORE_QTYPE_TXQ: Transmit Queue
+ * @PDS_CORE_QTYPE_EQ:Event Queue
+ * @PDS_CORE_QTYPE_MAX: Max queue type supported
+ */
+enum pds_core_logical_qtype {
+ PDS_CORE_QTYPE_ADMINQ = 0,
+ PDS_CORE_QTYPE_NOTIFYQ = 1,
+ PDS_CORE_QTYPE_RXQ = 2,
+ PDS_CORE_QTYPE_TXQ = 3,
+ PDS_CORE_QTYPE_EQ = 4,
+
+ PDS_CORE_QTYPE_MAX = 16 /* don't change - used in struct size */
+};
+
/**
* union pds_core_lif_config - LIF configuration
* @state: LIF state (enum pds_core_lif_state)
diff --git a/include/linux/pds/pds_common.h b/include/linux/pds/pds_common.h
index 2a0d1669cfd0..435c8e8161c2 100644
--- a/include/linux/pds/pds_common.h
+++ b/include/linux/pds/pds_common.h
@@ -41,27 +41,6 @@ enum pds_core_vif_types {
#define PDS_VDPA_DEV_NAME PDS_CORE_DRV_NAME "." PDS_DEV_TYPE_VDPA_STR
-#define PDS_CORE_IFNAMSIZ 16
-
-/**
- * enum pds_core_logical_qtype - Logical Queue Types
- * @PDS_CORE_QTYPE_ADMINQ:Administrative Queue
- * @PDS_CORE_QTYPE_NOTIFYQ: Notify Queue
- * @PDS_CORE_QTYPE_RXQ: Receive Queue
- * @PDS_CORE_QTYPE_TXQ: Transmit Queue
- * @PDS_CORE_QTYPE_EQ:Event Queue
- * @PDS_CORE_QTYPE_MAX: Max queue type supported
- */
-enum pds_core_logical_qtype {
- PDS_CORE_QTYPE_ADMINQ = 0,
- PDS_CORE_QTYPE_NOTIFYQ = 1,
- PDS_CORE_QTYPE_RXQ = 2,
- PDS_CORE_QTYPE_TXQ = 3,
- PDS_CORE_QTYPE_EQ = 4,
-
- PDS_CORE_QTYPE_MAX = 16 /* don't change - used in struct size */
-};
-
int pdsc_register_notify(struct notifier_block *nb);
void pdsc_unregister_notify(struct notifier_block *nb);
void *pdsc_get_pf_struct(struct pci_dev *vf_pdev);
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v5 virtio 07/11] pds_vdpa: virtio bar setup for vdpa
Prep and use the "modern" virtio bar utilities to get our
virtio config space ready.
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/aux_drv.c | 25 +
drivers/vdpa/pds/aux_drv.h | 3 +++
2 files changed, 28 insertions(+)
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
index aa748cf55d2b..0c4a135b1484 100644
--- a/drivers/vdpa/pds/aux_drv.c
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -4,6 +4,7 @@
#include
#include
#include
+#include
#include
#include
@@ -19,12 +20,22 @@ static const struct auxiliary_device_id pds_vdpa_id_table[]
= {
{},
};
+static int pds_vdpa_device_id_check(struct pci_dev *pdev)
+{
+ if (pdev->device != PCI_DEVICE_ID_PENSANDO_VDPA_VF ||
+ pdev->vendor != PCI_VENDOR_ID_PENSANDO)
+ return -ENODEV;
+
+ return PCI_DEVICE_ID_PENSANDO_VDPA_VF;
+}
+
static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
const struct auxiliary_device_id *id)
{
struct pds_auxiliary_dev *padev =
container_of(aux_dev, struct pds_auxiliary_dev, aux_dev);
+ struct device *dev = _dev->dev;
struct pds_vdpa_aux *vdpa_aux;
int err;
@@ -41,8 +52,21 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
if (err)
goto err_free_mem;
+ /* Find the virtio configuration */
+ vdpa_aux->vd_mdev.pci_dev = padev->vf_pdev;
+ vdpa_aux->vd_mdev.device_id_check = pds_vdpa_device_id_check;
+ vdpa_aux->vd_mdev.dma_mask = DMA_BIT_MASK(PDS_CORE_ADDR_LEN);
+ err = vp_modern_probe(_aux->vd_mdev);
+ if (err) {
+ dev_err(dev, "Unable to probe for virtio configuration: %pe\n",
+ ERR_PTR(err));
+ goto err_free_mgmt_info;
+ }
+
return 0;
+err_free_mgmt_info:
+ pci_free_irq_vectors(padev->vf_pdev);
err_free_mem:
kfree(vdpa_aux);
auxiliary_set_drvdata(aux_dev, NULL);
@@ -55,6 +79,7 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
struct device *dev = _dev->dev;
+ vp_modern_remove(_aux->vd_mdev);
pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
kfree(vdpa_aux);
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
index dcec782e79eb..99e0ff340bfa 100644
--- a/drivers/vdpa/pds/aux_drv.h
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -4,6 +4,8 @@
#ifndef _AUX_DRV_H_
#define _AUX_DRV_H_
+#include
+
#define PDS_VDPA_DRV_DESCRIPTION"AMD/Pensando vDPA VF Device Driver"
#define PDS_VDPA_DRV_NAME KBUILD_MODNAME
@@ -16,6 +18,7 @@ struct pds_vdpa_aux {
int vf_id;
struct dentry *dentry;
+ struct virtio_pci_modern_device vd_mdev;
int nintrs;
};
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v5 virtio 00/11] pds_vdpa driver
This patchset implements a new module for the AMD/Pensando DSC that supports vDPA services on PDS Core VF devices. This code is based on and depends on include files from the pds_core driver described here[0]. The pds_core driver creates the auxiliary_bus devices that this module connects to, and this creates vdpa devices for use by the vdpa module. The first version of this driver was a part of the original pds_core RFC [1] but has since been reworked to pull out the PCI driver and to make better use of the virtio and virtio_net configuration spaces made available by the DSC's PCI configuration. As the device development has progressed, the ability to rely on the virtio config spaces has grown. This patchset includes a modification to the existing vp_modern_probe() which implements overrides for the PCI device id check and the DMA mask. These are intended to be used with vendor vDPA devices that implement enough of the virtio config space to be used directly, but don't use the virtio device id. To use this module, enable the VFs and turn on the vDPA services in the pds_core PF, then use the 'vdpa' utility to create devices for use by virtio_vdpa or vhost_vdpa: echo 1 > /sys/bus/pci/drivers/pds_core/$PF_BDF/sriov_numvfs devlink dev param set pci/$PF_BDF name enable_vnet value true cmode runtime PDS_VDPA_MGMT=`vdpa mgmtdev show | grep vDPA | head -1 | cut -d: -f1` vdpa dev add name vdpa1 mgmtdev $PDS_VDPA_MGMT mac 00:11:22:33:44:55 [0] Link: https://lore.kernel.org/netdev/20230322185626.38758-1-shannon.nel...@amd.com/ [1] Link: https://lore.kernel.org/netdev/20221118225656.48309-1-snel...@pensando.io/ Changes: v5: - split dma_mask and device_id_check() into separate patches - simplify use of dma_mask into a single line change - changed test of VIRTIO_F_RING_PACKED to use BIT_ULL() v4: Link: https://lore.kernel.org/virtualization/20230425212602.1157-1-shannon.nel...@amd.com/ - rename device_id_check_override() to device_id_check() - make device_id_check() return the device_id found and checked - removed pds_vdpa.h, put its adminq changes into pds_adminq.h - added a patch to separate out the adminq changes - added a patch to move an adminq enum from pds_common.h to pds_adminq.h - moved adminq calls for get/set_vq_state into cmds.c - limit max_vqs by number of msix available - don't increment nintrs for CVQ, it should already be covered from max_vqs - pds_core API related rework following pds_core inclusion to net-next - use non-debugfs method to find PF pci address in pds_vdpa.rst instructions v3: Link: https://lore.kernel.org/netdev/20230330192313.62018-1-shannon.nel...@amd.com/ - added a patch to modify vp_modern_probe() such that specific device id and DMA mask overrides can be used - add pds_vdpa.rst into index file - dev_dbg instead of dev_err on most of the adminq commands - rework use of pds_vdpa_cmd_reset() and pds_vdpa_init_hw() for better firmware setup in start-stop-start scenarios - removed unused pds_vdpa_cmd_set_features(), we can rely on vp_modern_set_features() - remove unused hw_qtype and hw_qindex from pds_vdpa_vq_info - reworked debugfs print_feature_bits to also print unknown bits - changed use of PAGE_SIZE to local PDS_PAGE_SIZE to keep with FW layout needs without regard to kernel PAGE_SIZE configuration v2: https://lore.kernel.org/netdev/20230309013046.23523-1-shannon.nel...@amd.com/ - removed PCI driver code - replaced home-grown event listener with notifier - replaced many adminq uses with direct virtio_net config access - reworked irqs to follow virtio layout - removed local_mac_bit logic - replaced uses of devm_ interfaces as suggested in pds_core reviews - updated copyright strings to reflect the new owner Shannon Nelson (11): virtio: allow caller to override device id in vp_modern virtio: allow caller to override device DMA mask in vp_modern pds_vdpa: Add new vDPA driver for AMD/Pensando DSC pds_vdpa: move enum from common to adminq header pds_vdpa: new adminq entries pds_vdpa: get vdpa management info pds_vdpa: virtio bar setup for vdpa pds_vdpa: add vdpa config client commands pds_vdpa: add support for vdpa and vdpamgmt interfaces pds_vdpa: subscribe to the pds_core events pds_vdpa: pds_vdps.rst and Kconfig .../device_drivers/ethernet/amd/pds_vdpa.rst | 85 +++ .../device_drivers/ethernet/index.rst | 1 + MAINTAINERS | 4 + drivers/vdpa/Kconfig | 8 + drivers/vdpa/Makefile | 1 + drivers/vdpa/pds/Makefile | 10 + drivers/vdpa/pds/aux_drv.c| 140 drivers/vdpa/pds/aux_drv.h| 26 + drivers/vdpa/pds/cmds.c | 207 + drivers/vdpa/pds/cmds.h | 20 + drivers/vdpa/pds/debugfs.c| 287 +++ drivers/vdpa/pds/debugfs.h| 17 +
[PATCH v5 virtio 02/11] virtio: allow caller to override device DMA mask in vp_modern
To add a bit of vendor flexibility with various virtio based devices,
allow the caller to specify a different DMA mask. This adds a dma_mask
field to struct virtio_pci_modern_device. If defined by the driver,
this mask will be used in a call to dma_set_mask_and_coherent() instead
of the traditional DMA_BIT_MASK(64). This allows limiting the DMA space
on vendor devices with address limitations.
Signed-off-by: Shannon Nelson
---
drivers/virtio/virtio_pci_modern_dev.c | 3 ++-
include/linux/virtio_pci_modern.h | 3 +++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/drivers/virtio/virtio_pci_modern_dev.c
b/drivers/virtio/virtio_pci_modern_dev.c
index 9b2d6614de67..aad7d9296e77 100644
--- a/drivers/virtio/virtio_pci_modern_dev.c
+++ b/drivers/virtio/virtio_pci_modern_dev.c
@@ -268,7 +268,8 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
return -EINVAL;
}
- err = dma_set_mask_and_coherent(_dev->dev, DMA_BIT_MASK(64));
+ err = dma_set_mask_and_coherent(_dev->dev,
+ mdev->dma_mask ? : DMA_BIT_MASK(64));
if (err)
err = dma_set_mask_and_coherent(_dev->dev,
DMA_BIT_MASK(32));
diff --git a/include/linux/virtio_pci_modern.h
b/include/linux/virtio_pci_modern.h
index e7b1db1dd0bb..067ac1d789bc 100644
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
@@ -41,6 +41,9 @@ struct virtio_pci_modern_device {
/* optional check for vendor virtio device, returns dev_id or -ERRNO */
int (*device_id_check)(struct pci_dev *pdev);
+
+ /* optional mask for devices with limited DMA space */
+ u64 dma_mask;
};
/*
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH v4 virtio 01/10] virtio: allow caller to override device id and DMA mask
On 5/1/23 7:44 AM, Simon Horman wrote:
On Tue, Apr 25, 2023 at 02:25:53PM -0700, Shannon Nelson wrote:
To add a bit of flexibility with various virtio based devices, allow
the caller to specify a different device id and DMA mask. This adds
fields to struct virtio_pci_modern_device to specify an override device
id check and a DMA mask.
int (*device_id_check)(struct pci_dev *pdev);
If defined by the driver, this function will be called to check
that the PCI device is the vendor's expected device, and will
return the found device id to be stored in mdev->id.device.
This allows vendors with alternative vendor device ids to use
this library on their own device BAR.
u64 dma_mask;
If defined by the driver, this mask will be used in a call to
dma_set_mask_and_coherent() instead of the traditional
DMA_BIT_MASK(64). This allows limiting the DMA space on
vendor devices with address limitations.
Hi Shannon,
I don't feel strongly about this, but as there are two new features,
perhaps it would appropriate to have two patches.
Sure, I can respin and split these out to separate patches, and I'll
keep your verbosity notes below in mind :-). Yes, the kdoc would be a
good thing, but I'd like to keep the mission-creep to a minimum and come
back to that one separately.
sln
Signed-off-by: Shannon Nelson
---
drivers/virtio/virtio_pci_modern_dev.c | 37 +-
include/linux/virtio_pci_modern.h | 6 +
2 files changed, 31 insertions(+), 12 deletions(-)
diff --git a/drivers/virtio/virtio_pci_modern_dev.c
b/drivers/virtio/virtio_pci_modern_dev.c
index 869cb46bef96..1f2db76e8f91 100644
--- a/drivers/virtio/virtio_pci_modern_dev.c
+++ b/drivers/virtio/virtio_pci_modern_dev.c
@@ -218,21 +218,29 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
int err, common, isr, notify, device;
u32 notify_length;
u32 notify_offset;
+ int devid;
check_offsets();
- /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */
- if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)
- return -ENODEV;
-
- if (pci_dev->device < 0x1040) {
- /* Transitional devices: use the PCI subsystem device id as
- * virtio device id, same as legacy driver always did.
- */
- mdev->id.device = pci_dev->subsystem_device;
+ if (mdev->device_id_check) {
+ devid = mdev->device_id_check(pci_dev);
+ if (devid < 0)
+ return devid;
+ mdev->id.device = devid;
} else {
- /* Modern devices: simply use PCI device id, but start from
0x1040. */
- mdev->id.device = pci_dev->device - 0x1040;
+ /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */
+ if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)
+ return -ENODEV;
+
+ if (pci_dev->device < 0x1040) {
+ /* Transitional devices: use the PCI subsystem device id
as
+ * virtio device id, same as legacy driver always did.
+ */
+ mdev->id.device = pci_dev->subsystem_device;
+ } else {
+ /* Modern devices: simply use PCI device id, but start
from 0x1040. */
+ mdev->id.device = pci_dev->device - 0x1040;
+ }
}
mdev->id.vendor = pci_dev->subsystem_vendor;
The diff above is verbose, but looks good to me :)
@@ -260,7 +268,12 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
return -EINVAL;
}
- err = dma_set_mask_and_coherent(_dev->dev, DMA_BIT_MASK(64));
+ if (mdev->dma_mask)
+ err = dma_set_mask_and_coherent(_dev->dev,
+ mdev->dma_mask);
+ else
+ err = dma_set_mask_and_coherent(_dev->dev,
+ DMA_BIT_MASK(64));
Maybe it is nicer to avoid duplicating the function call, something like
this:
u64 dma_mask;
...
dma_mask = mdev->dma_mask ? : DMA_BIT_MASK(64);
err = dma_set_mask_and_coherent(_dev->dev, dma_mask);
or, without a local variable.
err = dma_set_mask_and_coherent(_dev->dev,
mdev->dma_mask ? : DMA_BIT_MASK(64));
if (err)
err = dma_set_mask_and_coherent(_dev->dev,
DMA_BIT_MASK(32));
diff --git a/include/linux/virtio_pci_modern.h
b/include/linux/virtio_pci_modern.h
index c4eeb79b0139..067ac1d789bc 100644
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
Maybe it would be good to add kdoc for struct virtio_pci_modern_device
at some point.
@@ -38,6 +38,12 @@ struct virtio_pci_modern_device {
int modern_bars;
Re: [PATCH v4 virtio 01/10] virtio: allow caller to override device id and DMA mask
On 4/25/23 7:09 PM, Xuan Zhuo wrote:
On Tue, 25 Apr 2023 14:25:53 -0700, Shannon Nelson
wrote:
To add a bit of flexibility with various virtio based devices, allow
the caller to specify a different device id and DMA mask. This adds
fields to struct virtio_pci_modern_device to specify an override device
id check and a DMA mask.
int (*device_id_check)(struct pci_dev *pdev);
If defined by the driver, this function will be called to check
that the PCI device is the vendor's expected device, and will
return the found device id to be stored in mdev->id.device.
This allows vendors with alternative vendor device ids to use
this library on their own device BAR.
u64 dma_mask;
If defined by the driver, this mask will be used in a call to
dma_set_mask_and_coherent() instead of the traditional
DMA_BIT_MASK(64). This allows limiting the DMA space on
vendor devices with address limitations.
Signed-off-by: Shannon Nelson
---
drivers/virtio/virtio_pci_modern_dev.c | 37 +-
include/linux/virtio_pci_modern.h | 6 +
2 files changed, 31 insertions(+), 12 deletions(-)
diff --git a/drivers/virtio/virtio_pci_modern_dev.c
b/drivers/virtio/virtio_pci_modern_dev.c
index 869cb46bef96..1f2db76e8f91 100644
--- a/drivers/virtio/virtio_pci_modern_dev.c
+++ b/drivers/virtio/virtio_pci_modern_dev.c
@@ -218,21 +218,29 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
int err, common, isr, notify, device;
u32 notify_length;
u32 notify_offset;
+ int devid;
check_offsets();
- /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */
- if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)
- return -ENODEV;
-
- if (pci_dev->device < 0x1040) {
- /* Transitional devices: use the PCI subsystem device id as
- * virtio device id, same as legacy driver always did.
- */
- mdev->id.device = pci_dev->subsystem_device;
+ if (mdev->device_id_check) {
+ devid = mdev->device_id_check(pci_dev);
+ if (devid < 0)
+ return devid;
I would want to know is there any other reason to return the errno?
How about return -ENODEV directly?
Because if device_id_check() is returning an errno, it is trying to
communicate some information about what went wrong, and I really get
annoyed when an intermediate layer stomps on the value and makes that
potentially useful information disappear.
sln
Thanks.
+ mdev->id.device = devid;
} else {
- /* Modern devices: simply use PCI device id, but start from
0x1040. */
- mdev->id.device = pci_dev->device - 0x1040;
+ /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */
+ if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)
+ return -ENODEV;
+
+ if (pci_dev->device < 0x1040) {
+ /* Transitional devices: use the PCI subsystem device id
as
+ * virtio device id, same as legacy driver always did.
+ */
+ mdev->id.device = pci_dev->subsystem_device;
+ } else {
+ /* Modern devices: simply use PCI device id, but start
from 0x1040. */
+ mdev->id.device = pci_dev->device - 0x1040;
+ }
}
mdev->id.vendor = pci_dev->subsystem_vendor;
@@ -260,7 +268,12 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
return -EINVAL;
}
- err = dma_set_mask_and_coherent(_dev->dev, DMA_BIT_MASK(64));
+ if (mdev->dma_mask)
+ err = dma_set_mask_and_coherent(_dev->dev,
+ mdev->dma_mask);
+ else
+ err = dma_set_mask_and_coherent(_dev->dev,
+ DMA_BIT_MASK(64));
if (err)
err = dma_set_mask_and_coherent(_dev->dev,
DMA_BIT_MASK(32));
diff --git a/include/linux/virtio_pci_modern.h
b/include/linux/virtio_pci_modern.h
index c4eeb79b0139..067ac1d789bc 100644
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
@@ -38,6 +38,12 @@ struct virtio_pci_modern_device {
int modern_bars;
struct virtio_device_id id;
+
+ /* optional check for vendor virtio device, returns dev_id or -ERRNO */
+ int (*device_id_check)(struct pci_dev *pdev);
+
+ /* optional mask for devices with limited DMA space */
+ u64 dma_mask;
};
/*
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v4 virtio 08/10] pds_vdpa: add support for vdpa and vdpamgmt interfaces
This is the vDPA device support, where we advertise that we can
support the virtio queues and deal with the configuration work
through the pds_core's adminq.
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/aux_drv.c | 15 +
drivers/vdpa/pds/aux_drv.h | 1 +
drivers/vdpa/pds/debugfs.c | 261 ++
drivers/vdpa/pds/debugfs.h | 5 +
drivers/vdpa/pds/vdpa_dev.c | 532 +++-
5 files changed, 813 insertions(+), 1 deletion(-)
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
index 0c4a135b1484..186e9ee22eb1 100644
--- a/drivers/vdpa/pds/aux_drv.c
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -63,8 +63,21 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
goto err_free_mgmt_info;
}
+ /* Let vdpa know that we can provide devices */
+ err = vdpa_mgmtdev_register(_aux->vdpa_mdev);
+ if (err) {
+ dev_err(dev, "%s: Failed to initialize vdpa_mgmt interface:
%pe\n",
+ __func__, ERR_PTR(err));
+ goto err_free_virtio;
+ }
+
+ pds_vdpa_debugfs_add_pcidev(vdpa_aux);
+ pds_vdpa_debugfs_add_ident(vdpa_aux);
+
return 0;
+err_free_virtio:
+ vp_modern_remove(_aux->vd_mdev);
err_free_mgmt_info:
pci_free_irq_vectors(padev->vf_pdev);
err_free_mem:
@@ -79,9 +92,11 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
struct device *dev = _dev->dev;
+ vdpa_mgmtdev_unregister(_aux->vdpa_mdev);
vp_modern_remove(_aux->vd_mdev);
pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
+ pds_vdpa_debugfs_del_vdpadev(vdpa_aux);
kfree(vdpa_aux);
auxiliary_set_drvdata(aux_dev, NULL);
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
index 99e0ff340bfa..26b75344156e 100644
--- a/drivers/vdpa/pds/aux_drv.h
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -13,6 +13,7 @@ struct pds_vdpa_aux {
struct pds_auxiliary_dev *padev;
struct vdpa_mgmt_dev vdpa_mdev;
+ struct pds_vdpa_device *pdsv;
struct pds_vdpa_ident ident;
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
index d91dceb07380..0ecd0e2ec6b9 100644
--- a/drivers/vdpa/pds/debugfs.c
+++ b/drivers/vdpa/pds/debugfs.c
@@ -10,6 +10,7 @@
#include
#include "aux_drv.h"
+#include "vdpa_dev.h"
#include "debugfs.h"
static struct dentry *dbfs_dir;
@@ -24,3 +25,263 @@ void pds_vdpa_debugfs_destroy(void)
debugfs_remove_recursive(dbfs_dir);
dbfs_dir = NULL;
}
+
+#define PRINT_SBIT_NAME(__seq, __f, __name) \
+ do {\
+ if ((__f) & (__name)) \
+ seq_printf(__seq, " %s",
[PATCH v4 virtio 09/10] pds_vdpa: subscribe to the pds_core events
Register for the pds_core's notification events, primarily to
find out when the FW has been reset so we can pass this on
back up the chain.
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/vdpa_dev.c | 68 -
drivers/vdpa/pds/vdpa_dev.h | 1 +
2 files changed, 68 insertions(+), 1 deletion(-)
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
index c3316f0faa0c..93b12f73423f 100644
--- a/drivers/vdpa/pds/vdpa_dev.c
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -21,6 +21,61 @@ static struct pds_vdpa_device *vdpa_to_pdsv(struct
vdpa_device *vdpa_dev)
return container_of(vdpa_dev, struct pds_vdpa_device, vdpa_dev);
}
+static int pds_vdpa_notify_handler(struct notifier_block *nb,
+ unsigned long ecode,
+ void *data)
+{
+ struct pds_vdpa_device *pdsv = container_of(nb, struct pds_vdpa_device,
nb);
+ struct device *dev = >vdpa_aux->padev->aux_dev.dev;
+
+ dev_dbg(dev, "%s: event code %lu\n", __func__, ecode);
+
+ /* Give the upper layers a hint that something interesting
+* may have happened. It seems that the only thing this
+* triggers in the virtio-net drivers above us is a check
+* of link status.
+*
+* We don't set the NEEDS_RESET flag for EVENT_RESET
+* because we're likely going through a recovery or
+* fw_update and will be back up and running soon.
+*/
+ if (ecode == PDS_EVENT_RESET || ecode == PDS_EVENT_LINK_CHANGE) {
+ if (pdsv->config_cb.callback)
+ pdsv->config_cb.callback(pdsv->config_cb.private);
+ }
+
+ return 0;
+}
+
+static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv)
+{
+ struct device *dev = >vdpa_aux->padev->aux_dev.dev;
+ struct notifier_block *nb = >nb;
+ int err;
+
+ if (!nb->notifier_call) {
+ nb->notifier_call = pds_vdpa_notify_handler;
+ err = pdsc_register_notify(nb);
+ if (err) {
+ nb->notifier_call = NULL;
+ dev_err(dev, "failed to register pds event handler:
%ps\n",
+ ERR_PTR(err));
+ return -EINVAL;
+ }
+ dev_dbg(dev, "pds event handler registered\n");
+ }
+
+ return 0;
+}
+
+static void pds_vdpa_unregister_event_handler(struct pds_vdpa_device *pdsv)
+{
+ if (pdsv->nb.notifier_call) {
+ pdsc_unregister_notify(>nb);
+ pdsv->nb.notifier_call = NULL;
+ }
+}
+
static int pds_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
u64 desc_addr, u64 driver_addr, u64
device_addr)
{
@@ -522,6 +577,12 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
pdsv->vdpa_dev.mdev = _aux->vdpa_mdev;
+ err = pds_vdpa_register_event_handler(pdsv);
+ if (err) {
+ dev_err(dev, "Failed to register for PDS events: %pe\n",
ERR_PTR(err));
+ goto err_unmap;
+ }
+
/* We use the _vdpa_register_device() call rather than the
* vdpa_register_device() to avoid a deadlock because our
* dev_add() is called with the vdpa_dev_lock already set
@@ -530,13 +591,15 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
err = _vdpa_register_device(>vdpa_dev, pdsv->num_vqs);
if (err) {
dev_err(dev, "Failed to register to vDPA bus: %pe\n",
ERR_PTR(err));
- goto err_unmap;
+ goto err_unevent;
}
pds_vdpa_debugfs_add_vdpadev(vdpa_aux);
return 0;
+err_unevent:
+ pds_vdpa_unregister_event_handler(pdsv);
err_unmap:
put_device(>vdpa_dev.dev);
vdpa_aux->pdsv = NULL;
@@ -546,8 +609,11 @@ static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev,
const char *name,
static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
struct vdpa_device *vdpa_dev)
{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
struct pds_vdpa_aux *vdpa_aux;
+ pds_vdpa_unregister_event_handler(pdsv);
+
vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
_vdpa_unregister_device(vdpa_dev);
diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h
index a21596f438c1..1650a2b08845 100644
--- a/drivers/vdpa/pds/vdpa_dev.h
+++ b/drivers/vdpa/pds/vdpa_dev.h
@@ -40,6 +40,7 @@ struct pds_vdpa_device {
u8 vdpa_index; /* rsvd for future subdevice use */
u8 num_vqs; /* num vqs in use */
struct vdpa_callback config_cb;
+ struct notifier_block nb;
};
int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux);
--
2.17.1
___
Virtualization mailing list
[PATCH v4 virtio 07/10] pds_vdpa: add vdpa config client commands
These are the adminq commands that will be needed for
setting up and using the vDPA device. There are a number
of commands defined in the FW's API, but by making use of
the FW's virtio BAR we only need a few of these commands
for vDPA support.
Signed-off-by: Shannon Nelson
---
Note: the previous version was Acked-by Jason Wang, but this has gone
through some rework of the adminq command API usage and added
two new commands. It is still essentially the same code, but
I've dropped the Acked-by for now.
drivers/vdpa/pds/Makefile | 1 +
drivers/vdpa/pds/cmds.c | 207
drivers/vdpa/pds/cmds.h | 20
drivers/vdpa/pds/vdpa_dev.h | 33 +-
4 files changed, 260 insertions(+), 1 deletion(-)
create mode 100644 drivers/vdpa/pds/cmds.c
create mode 100644 drivers/vdpa/pds/cmds.h
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
index 13b50394ec64..2e22418e3ab3 100644
--- a/drivers/vdpa/pds/Makefile
+++ b/drivers/vdpa/pds/Makefile
@@ -4,6 +4,7 @@
obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
pds_vdpa-y := aux_drv.o \
+ cmds.o \
vdpa_dev.o
pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/vdpa/pds/cmds.c b/drivers/vdpa/pds/cmds.c
new file mode 100644
index ..405711a0a0f8
--- /dev/null
+++ b/drivers/vdpa/pds/cmds.c
@@ -0,0 +1,207 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include "vdpa_dev.h"
+#include "aux_drv.h"
+#include "cmds.h"
+
+int pds_vdpa_init_hw(struct pds_vdpa_device *pdsv)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_init.opcode = PDS_VDPA_CMD_INIT,
+ .vdpa_init.vdpa_index = pdsv->vdpa_index,
+ .vdpa_init.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ /* Initialize the vdpa/virtio device */
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa_init),
+ , 0);
+ if (err)
+ dev_dbg(dev, "Failed to init hw, status %d: %pe\n",
+ comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_reset(struct pds_vdpa_device *pdsv)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa.opcode = PDS_VDPA_CMD_RESET,
+ .vdpa.vdpa_index = pdsv->vdpa_index,
+ .vdpa.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa), , 0);
+ if (err)
+ dev_dbg(dev, "Failed to reset hw, status %d: %pe\n",
+ comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_set_mac(struct pds_vdpa_device *pdsv, u8 *mac)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR,
+ .vdpa_setattr.vdpa_index = pdsv->vdpa_index,
+ .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ .vdpa_setattr.attr = PDS_VDPA_ATTR_MAC,
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ ether_addr_copy(cmd.vdpa_setattr.mac, mac);
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa_setattr),
+ , 0);
+ if (err)
+ dev_dbg(dev, "Failed to set mac address %pM, status %d: %pe\n",
+ mac, comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_set_max_vq_pairs(struct pds_vdpa_device *pdsv, u16 max_vqp)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = >aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR,
+ .vdpa_setattr.vdpa_index = pdsv->vdpa_index,
+ .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ .vdpa_setattr.attr = PDS_VDPA_ATTR_MAX_VQ_PAIRS,
+ .vdpa_setattr.max_vq_pairs = cpu_to_le16(max_vqp),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ err = pds_client_adminq_cmd(padev, , sizeof(cmd.vdpa_setattr),
+ , 0);
+ if (err)
+ dev_dbg(dev, "Failed to set max vq pairs %u, status %d: %pe\n",
+ max_vqp, comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_init_vq(struct pds_vdpa_device *pdsv, u16 qid,
+
[PATCH v4 virtio 10/10] pds_vdpa: pds_vdps.rst and Kconfig
Add the documentation and Kconfig entry for pds_vdpa driver.
Signed-off-by: Shannon Nelson
---
.../device_drivers/ethernet/amd/pds_vdpa.rst | 85 +++
.../device_drivers/ethernet/index.rst | 1 +
MAINTAINERS | 4 +
drivers/vdpa/Kconfig | 8 ++
4 files changed, 98 insertions(+)
create mode 100644
Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
diff --git a/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
b/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
new file mode 100644
index ..587927d3de92
--- /dev/null
+++ b/Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst
@@ -0,0 +1,85 @@
+.. SPDX-License-Identifier: GPL-2.0+
+.. note: can be edited and viewed with /usr/bin/formiko-vim
+
+==
+PCI vDPA driver for the AMD/Pensando(R) DSC adapter family
+==
+
+AMD/Pensando vDPA VF Device Driver
+
+Copyright(c) 2023 Advanced Micro Devices, Inc
+
+Overview
+
+
+The ``pds_vdpa`` driver is an auxiliary bus driver that supplies
+a vDPA device for use by the virtio network stack. It is used with
+the Pensando Virtual Function devices that offer vDPA and virtio queue
+services. It depends on the ``pds_core`` driver and hardware for the PF
+and VF PCI handling as well as for device configuration services.
+
+Using the device
+
+
+The ``pds_vdpa`` device is enabled via multiple configuration steps and
+depends on the ``pds_core`` driver to create and enable SR-IOV Virtual
+Function devices. After the VFs are enabled, we enable the vDPA service
+in the ``pds_core`` device to create the auxiliary devices used by pds_vdpa.
+
+Example steps:
+
+.. code-block:: bash
+
+ #!/bin/bash
+
+ modprobe pds_core
+ modprobe vdpa
+ modprobe pds_vdpa
+
+ PF_BDF=`ls /sys/module/pds_core/drivers/pci\:pds_core/*/sriov_numvfs | awk
-F / '{print $7}'`
+
+ # Enable vDPA VF auxiliary device(s) in the PF
+ devlink dev param set pci/$PF_BDF name enable_vnet cmode runtime value true
+
+ # Create a VF for vDPA use
+ echo 1 > /sys/bus/pci/drivers/pds_core/$PF_BDF/sriov_numvfs
+
+ # Find the vDPA services/devices available
+ PDS_VDPA_MGMT=`vdpa mgmtdev show | grep vDPA | head -1 | cut -d: -f1`
+
+ # Create a vDPA device for use in virtio network configurations
+ vdpa dev add name vdpa1 mgmtdev $PDS_VDPA_MGMT mac 00:11:22:33:44:55
+
+ # Set up an ethernet interface on the vdpa device
+ modprobe virtio_vdpa
+
+
+
+Enabling the driver
+===
+
+The driver is enabled via the standard kernel configuration system,
+using the make command::
+
+ make oldconfig/menuconfig/etc.
+
+The driver is located in the menu structure at:
+
+ -> Device Drivers
+-> Network device support (NETDEVICES [=y])
+ -> Ethernet driver support
+-> Pensando devices
+ -> Pensando Ethernet PDS_VDPA Support
+
+Support
+===
+
+For general Linux networking support, please use the netdev mailing
+list, which is monitored by Pensando personnel::
+
+ net...@vger.kernel.org
+
+For more specific support needs, please use the Pensando driver support
+email::
+
+ driv...@pensando.io
diff --git a/Documentation/networking/device_drivers/ethernet/index.rst
b/Documentation/networking/device_drivers/ethernet/index.rst
index 417ca514a4d0..94ecb67c0885 100644
--- a/Documentation/networking/device_drivers/ethernet/index.rst
+++ b/Documentation/networking/device_drivers/ethernet/index.rst
@@ -15,6 +15,7 @@ Contents:
amazon/ena
altera/altera_tse
amd/pds_core
+ amd/pds_vdpa
aquantia/atlantic
chelsio/cxgb
cirrus/cs89x0
diff --git a/MAINTAINERS b/MAINTAINERS
index 6ac562e0381e..93210a8ac74f 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -22148,6 +22148,10 @@ SNET DPU VIRTIO DATA PATH ACCELERATOR
R: Alvaro Karsz
F: drivers/vdpa/solidrun/
+PDS DSC VIRTIO DATA PATH ACCELERATOR
+R: Shannon Nelson
+F: drivers/vdpa/pds/
+
VIRTIO BALLOON
M: "Michael S. Tsirkin"
M: David Hildenbrand
diff --git a/drivers/vdpa/Kconfig b/drivers/vdpa/Kconfig
index cd6ad92f3f05..2ee1b288691d 100644
--- a/drivers/vdpa/Kconfig
+++ b/drivers/vdpa/Kconfig
@@ -116,4 +116,12 @@ config ALIBABA_ENI_VDPA
This driver includes a HW monitor device that
reads health values from the DPU.
+config PDS_VDPA
+ tristate "vDPA driver for AMD/Pensando DSC devices"
+ depends on PDS_CORE
+ help
+ vDPA network driver for AMD/Pensando's PDS Core devices.
+ With this driver, the VirtIO dataplane can be
+ offloaded to an AMD/Pensando DSC device.
+
endif # VDPA
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v4 virtio 06/10] pds_vdpa: virtio bar setup for vdpa
Prep and use the "modern" virtio bar utilities to get our
virtio config space ready.
Signed-off-by: Shannon Nelson
---
drivers/vdpa/pds/aux_drv.c | 25 +
drivers/vdpa/pds/aux_drv.h | 3 +++
2 files changed, 28 insertions(+)
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
index aa748cf55d2b..0c4a135b1484 100644
--- a/drivers/vdpa/pds/aux_drv.c
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -4,6 +4,7 @@
#include
#include
#include
+#include
#include
#include
@@ -19,12 +20,22 @@ static const struct auxiliary_device_id pds_vdpa_id_table[]
= {
{},
};
+static int pds_vdpa_device_id_check(struct pci_dev *pdev)
+{
+ if (pdev->device != PCI_DEVICE_ID_PENSANDO_VDPA_VF ||
+ pdev->vendor != PCI_VENDOR_ID_PENSANDO)
+ return -ENODEV;
+
+ return PCI_DEVICE_ID_PENSANDO_VDPA_VF;
+}
+
static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
const struct auxiliary_device_id *id)
{
struct pds_auxiliary_dev *padev =
container_of(aux_dev, struct pds_auxiliary_dev, aux_dev);
+ struct device *dev = _dev->dev;
struct pds_vdpa_aux *vdpa_aux;
int err;
@@ -41,8 +52,21 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
if (err)
goto err_free_mem;
+ /* Find the virtio configuration */
+ vdpa_aux->vd_mdev.pci_dev = padev->vf_pdev;
+ vdpa_aux->vd_mdev.device_id_check = pds_vdpa_device_id_check;
+ vdpa_aux->vd_mdev.dma_mask = DMA_BIT_MASK(PDS_CORE_ADDR_LEN);
+ err = vp_modern_probe(_aux->vd_mdev);
+ if (err) {
+ dev_err(dev, "Unable to probe for virtio configuration: %pe\n",
+ ERR_PTR(err));
+ goto err_free_mgmt_info;
+ }
+
return 0;
+err_free_mgmt_info:
+ pci_free_irq_vectors(padev->vf_pdev);
err_free_mem:
kfree(vdpa_aux);
auxiliary_set_drvdata(aux_dev, NULL);
@@ -55,6 +79,7 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
struct device *dev = _dev->dev;
+ vp_modern_remove(_aux->vd_mdev);
pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
kfree(vdpa_aux);
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
index dcec782e79eb..99e0ff340bfa 100644
--- a/drivers/vdpa/pds/aux_drv.h
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -4,6 +4,8 @@
#ifndef _AUX_DRV_H_
#define _AUX_DRV_H_
+#include
+
#define PDS_VDPA_DRV_DESCRIPTION"AMD/Pensando vDPA VF Device Driver"
#define PDS_VDPA_DRV_NAME KBUILD_MODNAME
@@ -16,6 +18,7 @@ struct pds_vdpa_aux {
int vf_id;
struct dentry *dentry;
+ struct virtio_pci_modern_device vd_mdev;
int nintrs;
};
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v4 virtio 02/10] pds_vdpa: Add new vDPA driver for AMD/Pensando DSC
This is the initial auxiliary driver framework for a new vDPA
device driver, an auxiliary_bus client of the pds_core driver.
The pds_core driver supplies the PCI services for the VF device
and for accessing the adminq in the PF device.
This patch adds the very basics of registering for the auxiliary
device and setting up debugfs entries.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/Makefile | 1 +
drivers/vdpa/pds/Makefile | 8
drivers/vdpa/pds/aux_drv.c | 83 ++
drivers/vdpa/pds/aux_drv.h | 15 ++
drivers/vdpa/pds/debugfs.c | 25 ++
drivers/vdpa/pds/debugfs.h | 12 +
include/linux/pds/pds_common.h | 2 +
7 files changed, 146 insertions(+)
create mode 100644 drivers/vdpa/pds/Makefile
create mode 100644 drivers/vdpa/pds/aux_drv.c
create mode 100644 drivers/vdpa/pds/aux_drv.h
create mode 100644 drivers/vdpa/pds/debugfs.c
create mode 100644 drivers/vdpa/pds/debugfs.h
diff --git a/drivers/vdpa/Makefile b/drivers/vdpa/Makefile
index 59396ff2a318..8f53c6f3cca7 100644
--- a/drivers/vdpa/Makefile
+++ b/drivers/vdpa/Makefile
@@ -7,3 +7,4 @@ obj-$(CONFIG_MLX5_VDPA) += mlx5/
obj-$(CONFIG_VP_VDPA)+= virtio_pci/
obj-$(CONFIG_ALIBABA_ENI_VDPA) += alibaba/
obj-$(CONFIG_SNET_VDPA) += solidrun/
+obj-$(CONFIG_PDS_VDPA) += pds/
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
new file mode 100644
index ..a9cd2f450ae1
--- /dev/null
+++ b/drivers/vdpa/pds/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# Copyright(c) 2023 Advanced Micro Devices, Inc
+
+obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
+
+pds_vdpa-y := aux_drv.o
+
+pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
new file mode 100644
index ..e4a0ad61ea22
--- /dev/null
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include "aux_drv.h"
+#include "debugfs.h"
+
+static const struct auxiliary_device_id pds_vdpa_id_table[] = {
+ { .name = PDS_VDPA_DEV_NAME, },
+ {},
+};
+
+static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
+ const struct auxiliary_device_id *id)
+
+{
+ struct pds_auxiliary_dev *padev =
+ container_of(aux_dev, struct pds_auxiliary_dev, aux_dev);
+ struct pds_vdpa_aux *vdpa_aux;
+
+ vdpa_aux = kzalloc(sizeof(*vdpa_aux), GFP_KERNEL);
+ if (!vdpa_aux)
+ return -ENOMEM;
+
+ vdpa_aux->padev = padev;
+ auxiliary_set_drvdata(aux_dev, vdpa_aux);
+
+ return 0;
+}
+
+static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
+{
+ struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
+ struct device *dev = _dev->dev;
+
+ kfree(vdpa_aux);
+ auxiliary_set_drvdata(aux_dev, NULL);
+
+ dev_info(dev, "Removed\n");
+}
+
+static struct auxiliary_driver pds_vdpa_driver = {
+ .name = PDS_DEV_TYPE_VDPA_STR,
+ .probe = pds_vdpa_probe,
+ .remove = pds_vdpa_remove,
+ .id_table = pds_vdpa_id_table,
+};
+
+static void __exit pds_vdpa_cleanup(void)
+{
+ auxiliary_driver_unregister(_vdpa_driver);
+
+ pds_vdpa_debugfs_destroy();
+}
+module_exit(pds_vdpa_cleanup);
+
+static int __init pds_vdpa_init(void)
+{
+ int err;
+
+ pds_vdpa_debugfs_create();
+
+ err = auxiliary_driver_register(_vdpa_driver);
+ if (err) {
+ pr_err("%s: aux driver register failed: %pe\n",
+ PDS_VDPA_DRV_NAME, ERR_PTR(err));
+ pds_vdpa_debugfs_destroy();
+ }
+
+ return err;
+}
+module_init(pds_vdpa_init);
+
+MODULE_DESCRIPTION(PDS_VDPA_DRV_DESCRIPTION);
+MODULE_AUTHOR("Advanced Micro Devices, Inc");
+MODULE_LICENSE("GPL");
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
new file mode 100644
index ..f1e99359424e
--- /dev/null
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#ifndef _AUX_DRV_H_
+#define _AUX_DRV_H_
+
+#define PDS_VDPA_DRV_DESCRIPTION"AMD/Pensando vDPA VF Device Driver"
+#define PDS_VDPA_DRV_NAME KBUILD_MODNAME
+
+struct pds_vdpa_aux {
+ struct pds_auxiliary_dev *padev;
+
+ struct dentry *dentry;
+};
+#endif /* _AUX_DRV_H_ */
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
new file mode 100644
index ..5be22fb7a76a
--- /dev/null
+++ b/drivers/vdpa/pds/debugfs.c
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+
+#include
+#include
+#include
+#include
+
+#include "aux_drv.h"
+#include "debugfs.h"
+
+static struct dentry *dbfs_dir;
+
+void
[PATCH v4 virtio 04/10] pds_vdpa: new adminq entries
Add new adminq definitions in support for vDPA operations.
Signed-off-by: Shannon Nelson
---
include/linux/pds/pds_adminq.h | 266 +
1 file changed, 266 insertions(+)
diff --git a/include/linux/pds/pds_adminq.h b/include/linux/pds/pds_adminq.h
index 61b0a8634e1a..c66ead725434 100644
--- a/include/linux/pds/pds_adminq.h
+++ b/include/linux/pds/pds_adminq.h
@@ -605,6 +605,257 @@ struct pds_core_q_init_comp {
u8 color;
};
+/*
+ * enum pds_vdpa_cmd_opcode - vDPA Device commands
+ */
+enum pds_vdpa_cmd_opcode {
+ PDS_VDPA_CMD_INIT = 48,
+ PDS_VDPA_CMD_IDENT = 49,
+ PDS_VDPA_CMD_RESET = 51,
+ PDS_VDPA_CMD_VQ_RESET = 52,
+ PDS_VDPA_CMD_VQ_INIT= 53,
+ PDS_VDPA_CMD_STATUS_UPDATE = 54,
+ PDS_VDPA_CMD_SET_FEATURES = 55,
+ PDS_VDPA_CMD_SET_ATTR = 56,
+ PDS_VDPA_CMD_VQ_SET_STATE = 57,
+ PDS_VDPA_CMD_VQ_GET_STATE = 58,
+};
+
+/**
+ * struct pds_vdpa_cmd - generic command
+ * @opcode:Opcode
+ * @vdpa_index:Index for vdpa subdevice
+ * @vf_id: VF id
+ */
+struct pds_vdpa_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+};
+
+/**
+ * struct pds_vdpa_init_cmd - INIT command
+ * @opcode:Opcode PDS_VDPA_CMD_INIT
+ * @vdpa_index: Index for vdpa subdevice
+ * @vf_id: VF id
+ */
+struct pds_vdpa_init_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+};
+
+/**
+ * struct pds_vdpa_ident - vDPA identification data
+ * @hw_features: vDPA features supported by device
+ * @max_vqs: max queues available (2 queues for a single queuepair)
+ * @max_qlen: log(2) of maximum number of descriptors
+ * @min_qlen: log(2) of minimum number of descriptors
+ *
+ * This struct is used in a DMA block that is set up for the PDS_VDPA_CMD_IDENT
+ * transaction. Set up the DMA block and send the address in the IDENT cmd
+ * data, the DSC will write the ident information, then we can remove the DMA
+ * block after reading the answer. If the completion status is 0, then there
+ * is valid information, else there was an error and the data should be
invalid.
+ */
+struct pds_vdpa_ident {
+ __le64 hw_features;
+ __le16 max_vqs;
+ __le16 max_qlen;
+ __le16 min_qlen;
+};
+
+/**
+ * struct pds_vdpa_ident_cmd - IDENT command
+ * @opcode:Opcode PDS_VDPA_CMD_IDENT
+ * @rsvd: Word boundary padding
+ * @vf_id: VF id
+ * @len: length of ident info DMA space
+ * @ident_pa: address for DMA of ident info (struct pds_vdpa_ident)
+ * only used for this transaction, then forgotten by DSC
+ */
+struct pds_vdpa_ident_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 vf_id;
+ __le32 len;
+ __le64 ident_pa;
+};
+
+/**
+ * struct pds_vdpa_status_cmd - STATUS_UPDATE command
+ * @opcode:Opcode PDS_VDPA_CMD_STATUS_UPDATE
+ * @vdpa_index: Index for vdpa subdevice
+ * @vf_id: VF id
+ * @status:new status bits
+ */
+struct pds_vdpa_status_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+ u8 status;
+};
+
+/**
+ * enum pds_vdpa_attr - List of VDPA device attributes
+ * @PDS_VDPA_ATTR_MAC: MAC address
+ * @PDS_VDPA_ATTR_MAX_VQ_PAIRS: Max virtqueue pairs
+ */
+enum pds_vdpa_attr {
+ PDS_VDPA_ATTR_MAC = 1,
+ PDS_VDPA_ATTR_MAX_VQ_PAIRS = 2,
+};
+
+/**
+ * struct pds_vdpa_setattr_cmd - SET_ATTR command
+ * @opcode:Opcode PDS_VDPA_CMD_SET_ATTR
+ * @vdpa_index:Index for vdpa subdevice
+ * @vf_id: VF id
+ * @attr: attribute to be changed (enum pds_vdpa_attr)
+ * @pad: Word boundary padding
+ * @mac: new mac address to be assigned as vdpa device address
+ * @max_vq_pairs: new limit of virtqueue pairs
+ */
+struct pds_vdpa_setattr_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+ u8 attr;
+ u8 pad[3];
+ union {
+ u8 mac[6];
+ __le16 max_vq_pairs;
+ } __packed;
+};
+
+/**
+ * struct pds_vdpa_vq_init_cmd - queue init command
+ * @opcode: Opcode PDS_VDPA_CMD_VQ_INIT
+ * @vdpa_index:Index for vdpa subdevice
+ * @vf_id: VF id
+ * @qid: Queue id (bit0 clear = rx, bit0 set = tx, qid=N is ctrlq)
+ * @len: log(2) of max descriptor count
+ * @desc_addr: DMA address of descriptor area
+ * @avail_addr:DMA address of available descriptors (aka driver area)
+ * @used_addr: DMA address of used descriptors (aka device area)
+ * @intr_index:interrupt index
+ */
+struct pds_vdpa_vq_init_cmd {
+ u8 opcode;
+ u8 vdpa_index;
+ __le16 vf_id;
+ __le16 qid;
+ __le16 len;
+ __le64 desc_addr;
+ __le64 avail_addr;
+ __le64 used_addr;
+ __le16 intr_index;
[PATCH v4 virtio 03/10] pds_vdpa: move enum from common to adminq header
The pds_core_logical_qtype enum and IFNAMSIZ are not needed
in the common PDS header, only needed when working with the
adminq, so move them to the adminq header.
Note: This patch might conflict with pds_vfio patches that are
in review, depending on which patchset gets pulled first.
Signed-off-by: Shannon Nelson
---
include/linux/pds/pds_adminq.h | 21 +
include/linux/pds/pds_common.h | 21 -
2 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/include/linux/pds/pds_adminq.h b/include/linux/pds/pds_adminq.h
index 98a60ce87b92..61b0a8634e1a 100644
--- a/include/linux/pds/pds_adminq.h
+++ b/include/linux/pds/pds_adminq.h
@@ -222,6 +222,27 @@ enum pds_core_lif_type {
PDS_CORE_LIF_TYPE_DEFAULT = 0,
};
+#define PDS_CORE_IFNAMSIZ 16
+
+/**
+ * enum pds_core_logical_qtype - Logical Queue Types
+ * @PDS_CORE_QTYPE_ADMINQ:Administrative Queue
+ * @PDS_CORE_QTYPE_NOTIFYQ: Notify Queue
+ * @PDS_CORE_QTYPE_RXQ: Receive Queue
+ * @PDS_CORE_QTYPE_TXQ: Transmit Queue
+ * @PDS_CORE_QTYPE_EQ:Event Queue
+ * @PDS_CORE_QTYPE_MAX: Max queue type supported
+ */
+enum pds_core_logical_qtype {
+ PDS_CORE_QTYPE_ADMINQ = 0,
+ PDS_CORE_QTYPE_NOTIFYQ = 1,
+ PDS_CORE_QTYPE_RXQ = 2,
+ PDS_CORE_QTYPE_TXQ = 3,
+ PDS_CORE_QTYPE_EQ = 4,
+
+ PDS_CORE_QTYPE_MAX = 16 /* don't change - used in struct size */
+};
+
/**
* union pds_core_lif_config - LIF configuration
* @state: LIF state (enum pds_core_lif_state)
diff --git a/include/linux/pds/pds_common.h b/include/linux/pds/pds_common.h
index 2a0d1669cfd0..435c8e8161c2 100644
--- a/include/linux/pds/pds_common.h
+++ b/include/linux/pds/pds_common.h
@@ -41,27 +41,6 @@ enum pds_core_vif_types {
#define PDS_VDPA_DEV_NAME PDS_CORE_DRV_NAME "." PDS_DEV_TYPE_VDPA_STR
-#define PDS_CORE_IFNAMSIZ 16
-
-/**
- * enum pds_core_logical_qtype - Logical Queue Types
- * @PDS_CORE_QTYPE_ADMINQ:Administrative Queue
- * @PDS_CORE_QTYPE_NOTIFYQ: Notify Queue
- * @PDS_CORE_QTYPE_RXQ: Receive Queue
- * @PDS_CORE_QTYPE_TXQ: Transmit Queue
- * @PDS_CORE_QTYPE_EQ:Event Queue
- * @PDS_CORE_QTYPE_MAX: Max queue type supported
- */
-enum pds_core_logical_qtype {
- PDS_CORE_QTYPE_ADMINQ = 0,
- PDS_CORE_QTYPE_NOTIFYQ = 1,
- PDS_CORE_QTYPE_RXQ = 2,
- PDS_CORE_QTYPE_TXQ = 3,
- PDS_CORE_QTYPE_EQ = 4,
-
- PDS_CORE_QTYPE_MAX = 16 /* don't change - used in struct size */
-};
-
int pdsc_register_notify(struct notifier_block *nb);
void pdsc_unregister_notify(struct notifier_block *nb);
void *pdsc_get_pf_struct(struct pci_dev *vf_pdev);
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v4 virtio 05/10] pds_vdpa: get vdpa management info
Find the vDPA management information from the DSC in order to
advertise it to the vdpa subsystem.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vdpa/pds/Makefile | 3 +-
drivers/vdpa/pds/aux_drv.c | 17 ++
drivers/vdpa/pds/aux_drv.h | 7 +++
drivers/vdpa/pds/debugfs.c | 1 +
drivers/vdpa/pds/vdpa_dev.c | 108
drivers/vdpa/pds/vdpa_dev.h | 15 +
6 files changed, 150 insertions(+), 1 deletion(-)
create mode 100644 drivers/vdpa/pds/vdpa_dev.c
create mode 100644 drivers/vdpa/pds/vdpa_dev.h
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
index a9cd2f450ae1..13b50394ec64 100644
--- a/drivers/vdpa/pds/Makefile
+++ b/drivers/vdpa/pds/Makefile
@@ -3,6 +3,7 @@
obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
-pds_vdpa-y := aux_drv.o
+pds_vdpa-y := aux_drv.o \
+ vdpa_dev.o
pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
index e4a0ad61ea22..aa748cf55d2b 100644
--- a/drivers/vdpa/pds/aux_drv.c
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -3,6 +3,7 @@
#include
#include
+#include
#include
#include
@@ -11,6 +12,7 @@
#include "aux_drv.h"
#include "debugfs.h"
+#include "vdpa_dev.h"
static const struct auxiliary_device_id pds_vdpa_id_table[] = {
{ .name = PDS_VDPA_DEV_NAME, },
@@ -24,15 +26,28 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
struct pds_auxiliary_dev *padev =
container_of(aux_dev, struct pds_auxiliary_dev, aux_dev);
struct pds_vdpa_aux *vdpa_aux;
+ int err;
vdpa_aux = kzalloc(sizeof(*vdpa_aux), GFP_KERNEL);
if (!vdpa_aux)
return -ENOMEM;
vdpa_aux->padev = padev;
+ vdpa_aux->vf_id = pci_iov_vf_id(padev->vf_pdev);
auxiliary_set_drvdata(aux_dev, vdpa_aux);
+ /* Get device ident info and set up the vdpa_mgmt_dev */
+ err = pds_vdpa_get_mgmt_info(vdpa_aux);
+ if (err)
+ goto err_free_mem;
+
return 0;
+
+err_free_mem:
+ kfree(vdpa_aux);
+ auxiliary_set_drvdata(aux_dev, NULL);
+
+ return err;
}
static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
@@ -40,6 +55,8 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
struct device *dev = _dev->dev;
+ pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
+
kfree(vdpa_aux);
auxiliary_set_drvdata(aux_dev, NULL);
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
index f1e99359424e..dcec782e79eb 100644
--- a/drivers/vdpa/pds/aux_drv.h
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -10,6 +10,13 @@
struct pds_vdpa_aux {
struct pds_auxiliary_dev *padev;
+ struct vdpa_mgmt_dev vdpa_mdev;
+
+ struct pds_vdpa_ident ident;
+
+ int vf_id;
struct dentry *dentry;
+
+ int nintrs;
};
#endif /* _AUX_DRV_H_ */
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
index 5be22fb7a76a..d91dceb07380 100644
--- a/drivers/vdpa/pds/debugfs.c
+++ b/drivers/vdpa/pds/debugfs.c
@@ -2,6 +2,7 @@
/* Copyright(c) 2023 Advanced Micro Devices, Inc */
#include
+#include
#include
#include
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
new file mode 100644
index ..0f0f0ab8b811
--- /dev/null
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include "vdpa_dev.h"
+#include "aux_drv.h"
+
+static struct virtio_device_id pds_vdpa_id_table[] = {
+ {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
+ {0},
+};
+
+static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
+ const struct vdpa_dev_set_config *add_config)
+{
+ return -EOPNOTSUPP;
+}
+
+static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
+struct vdpa_device *vdpa_dev)
+{
+}
+
+static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = {
+ .dev_add = pds_vdpa_dev_add,
+ .dev_del = pds_vdpa_dev_del
+};
+
+int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
+{
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_ident.opcode = PDS_VDPA_CMD_IDENT,
+ .vdpa_ident.vf_id = cpu_to_le16(vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ struct vdpa_mgmt_dev *mgmt;
+ struct pci_dev *pf_pdev;
+ struct device *pf_dev;
+ struct pci_dev *pdev;
+ dma_addr_t ident_pa;
+ struct device *dev;
+ u16 dev_intrs;
+ u16 max_vqs;
+ int err;
+
+ dev = _aux->padev->aux_dev.dev;
+ pdev = vdpa_aux->padev->vf_pdev;
+ mgmt = _aux->vdpa_mdev;
+
+ /* Get resource info through the PF's adminq. It
[PATCH v4 virtio 01/10] virtio: allow caller to override device id and DMA mask
To add a bit of flexibility with various virtio based devices, allow
the caller to specify a different device id and DMA mask. This adds
fields to struct virtio_pci_modern_device to specify an override device
id check and a DMA mask.
int (*device_id_check)(struct pci_dev *pdev);
If defined by the driver, this function will be called to check
that the PCI device is the vendor's expected device, and will
return the found device id to be stored in mdev->id.device.
This allows vendors with alternative vendor device ids to use
this library on their own device BAR.
u64 dma_mask;
If defined by the driver, this mask will be used in a call to
dma_set_mask_and_coherent() instead of the traditional
DMA_BIT_MASK(64). This allows limiting the DMA space on
vendor devices with address limitations.
Signed-off-by: Shannon Nelson
---
drivers/virtio/virtio_pci_modern_dev.c | 37 +-
include/linux/virtio_pci_modern.h | 6 +
2 files changed, 31 insertions(+), 12 deletions(-)
diff --git a/drivers/virtio/virtio_pci_modern_dev.c
b/drivers/virtio/virtio_pci_modern_dev.c
index 869cb46bef96..1f2db76e8f91 100644
--- a/drivers/virtio/virtio_pci_modern_dev.c
+++ b/drivers/virtio/virtio_pci_modern_dev.c
@@ -218,21 +218,29 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
int err, common, isr, notify, device;
u32 notify_length;
u32 notify_offset;
+ int devid;
check_offsets();
- /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */
- if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)
- return -ENODEV;
-
- if (pci_dev->device < 0x1040) {
- /* Transitional devices: use the PCI subsystem device id as
-* virtio device id, same as legacy driver always did.
-*/
- mdev->id.device = pci_dev->subsystem_device;
+ if (mdev->device_id_check) {
+ devid = mdev->device_id_check(pci_dev);
+ if (devid < 0)
+ return devid;
+ mdev->id.device = devid;
} else {
- /* Modern devices: simply use PCI device id, but start from
0x1040. */
- mdev->id.device = pci_dev->device - 0x1040;
+ /* We only own devices >= 0x1000 and <= 0x107f: leave the rest.
*/
+ if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)
+ return -ENODEV;
+
+ if (pci_dev->device < 0x1040) {
+ /* Transitional devices: use the PCI subsystem device
id as
+* virtio device id, same as legacy driver always did.
+*/
+ mdev->id.device = pci_dev->subsystem_device;
+ } else {
+ /* Modern devices: simply use PCI device id, but start
from 0x1040. */
+ mdev->id.device = pci_dev->device - 0x1040;
+ }
}
mdev->id.vendor = pci_dev->subsystem_vendor;
@@ -260,7 +268,12 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
return -EINVAL;
}
- err = dma_set_mask_and_coherent(_dev->dev, DMA_BIT_MASK(64));
+ if (mdev->dma_mask)
+ err = dma_set_mask_and_coherent(_dev->dev,
+ mdev->dma_mask);
+ else
+ err = dma_set_mask_and_coherent(_dev->dev,
+ DMA_BIT_MASK(64));
if (err)
err = dma_set_mask_and_coherent(_dev->dev,
DMA_BIT_MASK(32));
diff --git a/include/linux/virtio_pci_modern.h
b/include/linux/virtio_pci_modern.h
index c4eeb79b0139..067ac1d789bc 100644
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
@@ -38,6 +38,12 @@ struct virtio_pci_modern_device {
int modern_bars;
struct virtio_device_id id;
+
+ /* optional check for vendor virtio device, returns dev_id or -ERRNO */
+ int (*device_id_check)(struct pci_dev *pdev);
+
+ /* optional mask for devices with limited DMA space */
+ u64 dma_mask;
};
/*
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v4 virtio 00/10] pds_vdpa driver
This patchset implements a new module for the AMD/Pensando DSC that supports vDPA services on PDS Core VF devices. This code is based on and depends on include files from the pds_core driver described here[0]. The pds_core driver creates the auxiliary_bus devices that this module connects to, and this creates vdpa devices for use by the vdpa module. The first version of this driver was a part of the original pds_core RFC [1] but has since been reworked to pull out the PCI driver and to make better use of the virtio and virtio_net configuration spaces made available by the DSC's PCI configuration. As the device development has progressed, the ability to rely on the virtio config spaces has grown. This patchset includes a modification to the existing vp_modern_probe() which implements overrides for the PCI device id check and the DMA mask. These are intended to be used with vendor vDPA devices that implement enough of the virtio config space to be used directly, but don't use the virtio device id. To use this module, enable the VFs and turn on the vDPA services in the pds_core PF, then use the 'vdpa' utility to create devices for use by virtio_vdpa or vhost_vdpa: echo 1 > /sys/bus/pci/drivers/pds_core/$PF_BDF/sriov_numvfs devlink dev param set pci/$PF_BDF name enable_vnet value true cmode runtime PDS_VDPA_MGMT=`vdpa mgmtdev show | grep vDPA | head -1 | cut -d: -f1` vdpa dev add name vdpa1 mgmtdev $PDS_VDPA_MGMT mac 00:11:22:33:44:55 [0] Link: https://lore.kernel.org/netdev/20230322185626.38758-1-shannon.nel...@amd.com/ [1] Link: https://lore.kernel.org/netdev/20221118225656.48309-1-snel...@pensando.io/ Changes: v4: - rename device_id_check_override() to device_id_check() - make device_id_check() return the device_id found and checked - removed pds_vdpa.h, put its adminq changes into pds_adminq.h - added a patch to separate out the adminq changes - added a patch to move an adminq enum from pds_common.h to pds_adminq.h - moved adminq calls for get/set_vq_state into cmds.c - limit max_vqs by number of msix available - don't increment nintrs for CVQ, it should already be covered from max_vqs - pds_core API related rework following pds_core inclusion to net-next - use non-debugfs method to find PF pci address in pds_vdpa.rst instructions v3: Link: https://lore.kernel.org/netdev/20230330192313.62018-1-shannon.nel...@amd.com/ - added a patch to modify vp_modern_probe() such that specific device id and DMA mask overrides can be used - add pds_vdpa.rst into index file - dev_dbg instead of dev_err on most of the adminq commands - rework use of pds_vdpa_cmd_reset() and pds_vdpa_init_hw() for better firmware setup in start-stop-start scenarios - removed unused pds_vdpa_cmd_set_features(), we can rely on vp_modern_set_features() - remove unused hw_qtype and hw_qindex from pds_vdpa_vq_info - reworked debugfs print_feature_bits to also print unknown bits - changed use of PAGE_SIZE to local PDS_PAGE_SIZE to keep with FW layout needs without regard to kernel PAGE_SIZE configuration v2: https://lore.kernel.org/netdev/20230309013046.23523-1-shannon.nel...@amd.com/ - removed PCI driver code - replaced home-grown event listener with notifier - replaced many adminq uses with direct virtio_net config access - reworked irqs to follow virtio layout - removed local_mac_bit logic - replaced uses of devm_ interfaces as suggested in pds_core reviews - updated copyright strings to reflect the new owner Shannon Nelson (10): virtio: allow caller to override device id and DMA mask pds_vdpa: Add new vDPA driver for AMD/Pensando DSC pds_vdpa: move enum from common to adminq header pds_vdpa: new adminq entries pds_vdpa: get vdpa management info pds_vdpa: virtio bar setup for vdpa pds_vdpa: add vdpa config client commands pds_vdpa: add support for vdpa and vdpamgmt interfaces pds_vdpa: subscribe to the pds_core events pds_vdpa: pds_vdps.rst and Kconfig .../device_drivers/ethernet/amd/pds_vdpa.rst | 85 +++ .../device_drivers/ethernet/index.rst | 1 + MAINTAINERS | 4 + drivers/vdpa/Kconfig | 8 + drivers/vdpa/Makefile | 1 + drivers/vdpa/pds/Makefile | 10 + drivers/vdpa/pds/aux_drv.c| 140 drivers/vdpa/pds/aux_drv.h| 26 + drivers/vdpa/pds/cmds.c | 207 + drivers/vdpa/pds/cmds.h | 20 + drivers/vdpa/pds/debugfs.c| 287 +++ drivers/vdpa/pds/debugfs.h| 17 + drivers/vdpa/pds/vdpa_dev.c | 704 ++ drivers/vdpa/pds/vdpa_dev.h | 47 ++ drivers/virtio/virtio_pci_modern_dev.c| 37 +- include/linux/pds/pds_adminq.h| 287 +++ include/linux/pds/pds_common.h| 21 +- include/linux/virtio_pci_modern.h | 6
Re: [PATCH v2 0/3] vhost_vdpa: better PACKED support
On 4/24/23 11:08 PM, Michael S. Tsirkin wrote: On Mon, Apr 24, 2023 at 03:50:28PM -0700, Shannon Nelson wrote: While testing our vDPA driver with QEMU we found that vhost_vdpa was missing some support for PACKED VQs. Adding these helped us get further in our testing. The first patch makes sure that the vhost_vdpa vqs are given the feature flags, as done in other vhost variants. The second and third patches use the feature flags to properly format and pass set/get_vq requests. This missed the merge window most likely, unless there's a reason to send a second pull towards the end of it. We'll see. Understood - we'll keep our fingers crossed for good luck :-) Thanks for the help. sln v1: - included wrap counter in packing answers for VHOST_GET_VRING_BASE - added comments to vhost_virtqueue fields last_avail_idx and last_used_idx v0: https://lists.linuxfoundation.org/pipermail/virtualization/2023-April/066253.html - initial version Shannon Nelson (3): vhost_vdpa: tell vqs about the negotiated vhost: support PACKED when setting-getting vring_base vhost_vdpa: support PACKED when setting-getting vring_base drivers/vhost/vdpa.c | 34 ++ drivers/vhost/vhost.c | 18 +- drivers/vhost/vhost.h | 8 ++-- 3 files changed, 49 insertions(+), 11 deletions(-) -- 2.17.1 ___ Virtualization mailing list Virtualization@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v2 0/3] vhost_vdpa: better PACKED support
While testing our vDPA driver with QEMU we found that vhost_vdpa was missing some support for PACKED VQs. Adding these helped us get further in our testing. The first patch makes sure that the vhost_vdpa vqs are given the feature flags, as done in other vhost variants. The second and third patches use the feature flags to properly format and pass set/get_vq requests. v1: - included wrap counter in packing answers for VHOST_GET_VRING_BASE - added comments to vhost_virtqueue fields last_avail_idx and last_used_idx v0: https://lists.linuxfoundation.org/pipermail/virtualization/2023-April/066253.html - initial version Shannon Nelson (3): vhost_vdpa: tell vqs about the negotiated vhost: support PACKED when setting-getting vring_base vhost_vdpa: support PACKED when setting-getting vring_base drivers/vhost/vdpa.c | 34 ++ drivers/vhost/vhost.c | 18 +- drivers/vhost/vhost.h | 8 ++-- 3 files changed, 49 insertions(+), 11 deletions(-) -- 2.17.1 ___ Virtualization mailing list Virtualization@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v2 1/3] vhost_vdpa: tell vqs about the negotiated
As is done in the net, iscsi, and vsock vhost support, let the vdpa vqs
know about the features that have been negotiated. This allows vhost
to more safely make decisions based on the features, such as when using
PACKED vs split queues.
Signed-off-by: Shannon Nelson
Acked-by: Jason Wang
---
drivers/vhost/vdpa.c | 13 +
1 file changed, 13 insertions(+)
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index 7be9d9d8f01c..599b8cc238c7 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -385,7 +385,10 @@ static long vhost_vdpa_set_features(struct vhost_vdpa *v,
u64 __user *featurep)
{
struct vdpa_device *vdpa = v->vdpa;
const struct vdpa_config_ops *ops = vdpa->config;
+ struct vhost_dev *d = >vdev;
+ u64 actual_features;
u64 features;
+ int i;
/*
* It's not allowed to change the features after they have
@@ -400,6 +403,16 @@ static long vhost_vdpa_set_features(struct vhost_vdpa *v,
u64 __user *featurep)
if (vdpa_set_features(vdpa, features))
return -EINVAL;
+ /* let the vqs know what has been configured */
+ actual_features = ops->get_driver_features(vdpa);
+ for (i = 0; i < d->nvqs; ++i) {
+ struct vhost_virtqueue *vq = d->vqs[i];
+
+ mutex_lock(>mutex);
+ vq->acked_features = actual_features;
+ mutex_unlock(>mutex);
+ }
+
return 0;
}
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v2 3/3] vhost_vdpa: support PACKED when setting-getting vring_base
Use the right structs for PACKED or split vqs when setting and
getting the vring base.
Signed-off-by: Shannon Nelson
---
drivers/vhost/vdpa.c | 21 +
1 file changed, 17 insertions(+), 4 deletions(-)
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index 599b8cc238c7..fe392b67d5be 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -585,7 +585,14 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v,
unsigned int cmd,
if (r)
return r;
- vq->last_avail_idx = vq_state.split.avail_index;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
+ vq->last_avail_idx = vq_state.packed.last_avail_idx |
+
(vq_state.packed.last_avail_counter << 15);
+ vq->last_used_idx = vq_state.packed.last_used_idx |
+ (vq_state.packed.last_used_counter
<< 15);
+ } else {
+ vq->last_avail_idx = vq_state.split.avail_index;
+ }
break;
}
@@ -603,9 +610,15 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v,
unsigned int cmd,
break;
case VHOST_SET_VRING_BASE:
- vq_state.split.avail_index = vq->last_avail_idx;
- if (ops->set_vq_state(vdpa, idx, _state))
- r = -EINVAL;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
+ vq_state.packed.last_avail_idx = vq->last_avail_idx &
0x7fff;
+ vq_state.packed.last_avail_counter =
!!(vq->last_avail_idx & 0x8000);
+ vq_state.packed.last_used_idx = vq->last_used_idx &
0x7fff;
+ vq_state.packed.last_used_counter =
!!(vq->last_used_idx & 0x8000);
+ } else {
+ vq_state.split.avail_index = vq->last_avail_idx;
+ }
+ r = ops->set_vq_state(vdpa, idx, _state);
break;
case VHOST_SET_VRING_CALL:
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH v2 2/3] vhost: support PACKED when setting-getting vring_base
Use the right structs for PACKED or split vqs when setting and
getting the vring base.
Signed-off-by: Shannon Nelson
---
drivers/vhost/vhost.c | 18 +-
drivers/vhost/vhost.h | 8 ++--
2 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index f11bdbe4c2c5..f64efda48f21 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -1633,17 +1633,25 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned
int ioctl, void __user *arg
r = -EFAULT;
break;
}
- if (s.num > 0x) {
- r = -EINVAL;
- break;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
+ vq->last_avail_idx = s.num & 0x;
+ vq->last_used_idx = (s.num >> 16) & 0x;
+ } else {
+ if (s.num > 0x) {
+ r = -EINVAL;
+ break;
+ }
+ vq->last_avail_idx = s.num;
}
- vq->last_avail_idx = s.num;
/* Forget the cached index value. */
vq->avail_idx = vq->last_avail_idx;
break;
case VHOST_GET_VRING_BASE:
s.index = idx;
- s.num = vq->last_avail_idx;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED))
+ s.num = (u32)vq->last_avail_idx |
((u32)vq->last_used_idx << 16);
+ else
+ s.num = vq->last_avail_idx;
if (copy_to_user(argp, , sizeof s))
r = -EFAULT;
break;
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index 1647b750169c..6f73f29d5979 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -85,13 +85,17 @@ struct vhost_virtqueue {
/* The routine to call when the Guest pings us, or timeout. */
vhost_work_fn_t handle_kick;
- /* Last available index we saw. */
+ /* Last available index we saw.
+* Values are limited to 0x7fff, and the high bit is used as
+* a wrap counter when using VIRTIO_F_RING_PACKED. */
u16 last_avail_idx;
/* Caches available index value from user. */
u16 avail_idx;
- /* Last index we used. */
+ /* Last index we used.
+* Values are limited to 0x7fff, and the high bit is used as
+* a wrap counter when using VIRTIO_F_RING_PACKED. */
u16 last_used_idx;
/* Used flags */
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH 3/3] vhost_vdpa: support PACKED when setting-getting vring_base
On 4/22/23 11:40 PM, Jason Wang wrote:
On Sat, Apr 22, 2023 at 3:57 AM Shannon Nelson wrote:
Use the right structs for PACKED or split vqs when setting and
getting the vring base.
Signed-off-by: Shannon Nelson
---
drivers/vhost/vdpa.c | 19 +++
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index 599b8cc238c7..2543ae9d5939 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -585,7 +585,12 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v,
unsigned int cmd,
if (r)
return r;
- vq->last_avail_idx = vq_state.split.avail_index;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
+ vq->last_avail_idx = vq_state.packed.last_avail_idx;
+ vq->last_used_idx = vq_state.packed.last_used_idx;
I wonder if the compiler will guarantee a piggyback for the wrap
counter here? If not, should we explicitly packing wrap counter?
Yes, good catch, I'll add the wrap counter.
sln
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH 2/3] vhost: support PACKED when setting-getting vring_base
On 4/22/23 11:36 PM, Jason Wang wrote:
On Sat, Apr 22, 2023 at 3:57 AM Shannon Nelson wrote:
Use the right structs for PACKED or split vqs when setting and
getting the vring base.
Signed-off-by: Shannon Nelson
---
drivers/vhost/vhost.c | 18 +-
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index f11bdbe4c2c5..f64efda48f21 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -1633,17 +1633,25 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned
int ioctl, void __user *arg
r = -EFAULT;
break;
}
- if (s.num > 0x) {
- r = -EINVAL;
- break;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
+ vq->last_avail_idx = s.num & 0x;
+ vq->last_used_idx = (s.num >> 16) & 0x;
I think we need to tweak the comment around last_avail_idx and last_used_idx:
/* Last available index we saw. */
u16 last_avail_idx;
/* Last index we used. */
u16 last_used_idx;
To describe that it contains wrap counters as well in the case of
packed virtqueue
Sure, I can add into the comments that these counters are limited to
0x7fff and the high bits are used for a wrap counter/flag.
or maybe it's time to rename them (since they are
invented for split virtqueue).
Should we change them to bitfields as in struct vdpa_vq_state_packed?
Or perhaps just add new fields for bool/u16/u8 last_avail_counter and
last_used_counter?
That might be a later patch in order to also deal with whatever fallout
happens from a new name.
sln
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH 3/3] vhost_vdpa: support PACKED when setting-getting vring_base
Use the right structs for PACKED or split vqs when setting and
getting the vring base.
Signed-off-by: Shannon Nelson
---
drivers/vhost/vdpa.c | 19 +++
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index 599b8cc238c7..2543ae9d5939 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -585,7 +585,12 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v,
unsigned int cmd,
if (r)
return r;
- vq->last_avail_idx = vq_state.split.avail_index;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
+ vq->last_avail_idx = vq_state.packed.last_avail_idx;
+ vq->last_used_idx = vq_state.packed.last_used_idx;
+ } else {
+ vq->last_avail_idx = vq_state.split.avail_index;
+ }
break;
}
@@ -603,9 +608,15 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v,
unsigned int cmd,
break;
case VHOST_SET_VRING_BASE:
- vq_state.split.avail_index = vq->last_avail_idx;
- if (ops->set_vq_state(vdpa, idx, _state))
- r = -EINVAL;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
+ vq_state.packed.last_avail_idx = vq->last_avail_idx &
0x7fff;
+ vq_state.packed.last_avail_counter =
!!(vq->last_avail_idx & 0x8000);
+ vq_state.packed.last_used_idx = vq->last_used_idx &
0x7fff;
+ vq_state.packed.last_used_counter =
!!(vq->last_used_idx & 0x8000);
+ } else {
+ vq_state.split.avail_index = vq->last_avail_idx;
+ }
+ r = ops->set_vq_state(vdpa, idx, _state);
break;
case VHOST_SET_VRING_CALL:
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH 1/3] vhost_vdpa: tell vqs about the negotiated
As is done in the net, iscsi, and vsock vhost support, let the vdpa vqs
know about the features that have been negotiated. This allows vhost
to more safely make decisions based on the features, such as when using
PACKED vs split queues.
Signed-off-by: Shannon Nelson
---
drivers/vhost/vdpa.c | 13 +
1 file changed, 13 insertions(+)
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index 7be9d9d8f01c..599b8cc238c7 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -385,7 +385,10 @@ static long vhost_vdpa_set_features(struct vhost_vdpa *v,
u64 __user *featurep)
{
struct vdpa_device *vdpa = v->vdpa;
const struct vdpa_config_ops *ops = vdpa->config;
+ struct vhost_dev *d = >vdev;
+ u64 actual_features;
u64 features;
+ int i;
/*
* It's not allowed to change the features after they have
@@ -400,6 +403,16 @@ static long vhost_vdpa_set_features(struct vhost_vdpa *v,
u64 __user *featurep)
if (vdpa_set_features(vdpa, features))
return -EINVAL;
+ /* let the vqs know what has been configured */
+ actual_features = ops->get_driver_features(vdpa);
+ for (i = 0; i < d->nvqs; ++i) {
+ struct vhost_virtqueue *vq = d->vqs[i];
+
+ mutex_lock(>mutex);
+ vq->acked_features = actual_features;
+ mutex_unlock(>mutex);
+ }
+
return 0;
}
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH 0/3] vhost_vdpa: better PACKED support
While testing our vDPA driver with QEMU we found that vhost_vdpa seems to be missing some support for PACKED VQs. Adding these helped us get further in our testing. The first patch makes sure that the vhost_vdpa vqs are given the feature flags, as done in other vhost variants. The second and third patches use the feature flags to properly format and pass set/get_vq requests. Shannon Nelson (3): vhost_vdpa: tell vqs about the negotiated vhost: support PACKED when setting-getting vring_base vhost_vdpa: support PACKED when setting-getting vring_base drivers/vhost/vdpa.c | 32 drivers/vhost/vhost.c | 18 +- 2 files changed, 41 insertions(+), 9 deletions(-) -- 2.17.1 ___ Virtualization mailing list Virtualization@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/virtualization
[PATCH 2/3] vhost: support PACKED when setting-getting vring_base
Use the right structs for PACKED or split vqs when setting and
getting the vring base.
Signed-off-by: Shannon Nelson
---
drivers/vhost/vhost.c | 18 +-
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index f11bdbe4c2c5..f64efda48f21 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -1633,17 +1633,25 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned
int ioctl, void __user *arg
r = -EFAULT;
break;
}
- if (s.num > 0x) {
- r = -EINVAL;
- break;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
+ vq->last_avail_idx = s.num & 0x;
+ vq->last_used_idx = (s.num >> 16) & 0x;
+ } else {
+ if (s.num > 0x) {
+ r = -EINVAL;
+ break;
+ }
+ vq->last_avail_idx = s.num;
}
- vq->last_avail_idx = s.num;
/* Forget the cached index value. */
vq->avail_idx = vq->last_avail_idx;
break;
case VHOST_GET_VRING_BASE:
s.index = idx;
- s.num = vq->last_avail_idx;
+ if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED))
+ s.num = (u32)vq->last_avail_idx |
((u32)vq->last_used_idx << 16);
+ else
+ s.num = vq->last_avail_idx;
if (copy_to_user(argp, , sizeof s))
r = -EFAULT;
break;
--
2.17.1
___
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
