From: Jason Gunthorpe <[email protected]> The SGT (Scatter Gather Table) DMA mapping type represents the existing sg_table/scatterlist based DMA mapping. It provides a sg_table based map/unmap interface that exactly matches how things work today.
dma_buf_sgt_exp_compat_match will be used in the next patch to transparently wrap an unaware exporter with a mapping type. The SGT type handles the allow_peer2peer flag directly through matching logic. The importer indicates if it is willing to accept peer2peer and the exporter indicates if it requires peer2peer. A required peer2peer exporter will not match to an importer that does not accept peer2peer. Signed-off-by: Jason Gunthorpe <[email protected]> Signed-off-by: Alistair Popple <[email protected]> --- drivers/dma-buf/dma-buf-mapping.c | 104 +++++++++++++++++++++++++++++++ include/linux/dma-buf-mapping.h | 127 ++++++++++++++++++++++++++++++++++++++ include/linux/dma-buf.h | 19 +++++- 3 files changed, 249 insertions(+), 1 deletion(-) diff --git a/drivers/dma-buf/dma-buf-mapping.c b/drivers/dma-buf/dma-buf-mapping.c index d27f02610c2a..4eb0ea930900 100644 --- a/drivers/dma-buf/dma-buf-mapping.c +++ b/drivers/dma-buf/dma-buf-mapping.c @@ -6,6 +6,7 @@ #include <linux/dma-buf-mapping.h> #include <linux/dma-resv.h> #include <linux/dma-buf.h> +#include <linux/seq_file.h> #include <linux/overflow.h> #include <linux/align.h> @@ -322,3 +323,106 @@ int dma_buf_match_mapping(struct dma_buf_match_args *args, return 0; } EXPORT_SYMBOL_NS_GPL(dma_buf_match_mapping, "DMA_BUF"); + +static int dma_buf_sgt_match(struct dma_buf_match_args *args, + const struct dma_buf_mapping_match *exp, + const struct dma_buf_mapping_match *imp) +{ + switch (exp->sgt_data.exporter_requires_p2p) { + case DMA_SGT_NO_P2P: + return 0; + case DMA_SGT_EXPORTER_REQUIRES_P2P: + if (imp->sgt_data.importer_accepts_p2p != + DMA_SGT_IMPORTER_ACCEPTS_P2P) + return -EOPNOTSUPP; + return 0; + case DMA_SGT_EXPORTER_REQUIRES_P2P_DISTANCE: + if (WARN_ON(!exp->sgt_data.exporting_p2p_device) || + imp->sgt_data.importer_accepts_p2p != + DMA_SGT_IMPORTER_ACCEPTS_P2P) + return -EOPNOTSUPP; + /* + * Not verbose: failing to match P2P is normal when the + * exporter also offers a system memory type. + */ + if (pci_p2pdma_distance(exp->sgt_data.exporting_p2p_device, + args->dev, false) < 0) + return -EOPNOTSUPP; + return 0; + } + return 0; +} + +static inline void +dma_buf_sgt_finish_match(struct dma_buf_match_args *args, + const struct dma_buf_mapping_match *exp, + const struct dma_buf_mapping_match *imp, + struct dma_buf_mapping_match *match) +{ + struct dma_buf_attachment *attach = args->attach; + + *match = (struct dma_buf_mapping_match) { + .type = &dma_buf_mapping_sgt_type, + .exp_ops = exp->exp_ops, + .sgt_data = { + .importing_dma_device = args->dev, + /* exporting_p2p_device is left opaque */ + .importer_accepts_p2p = imp->sgt_data.importer_accepts_p2p, + .exporter_requires_p2p = exp->sgt_data.exporter_requires_p2p, + }, + }; + + /* + * Setup the SGT type variables stored in attach because importers and + * exporters that do not natively use mappings expect them to be there. + * When converting to use mappings users should use the match versions + * of these instead. + */ + attach->dev = args->dev; + attach->peer2peer = match->sgt_data.importer_accepts_p2p == + DMA_SGT_IMPORTER_ACCEPTS_P2P; +} + +static void dma_buf_sgt_debugfs_dump(struct seq_file *s, + struct dma_buf_attachment *attach) +{ + seq_printf(s, " %s", dev_name(dma_buf_sgt_dma_device(attach))); +} + +struct dma_buf_mapping_type dma_buf_mapping_sgt_type = { + .name = "DMA Mapped Scatter Gather Table", + .match = dma_buf_sgt_match, + .finish_match = dma_buf_sgt_finish_match, + .debugfs_dump = dma_buf_sgt_debugfs_dump, +}; +EXPORT_SYMBOL_NS_GPL(dma_buf_mapping_sgt_type, "DMA_BUF"); + +static struct sg_table * +dma_buf_sgt_compat_map_dma_buf(struct dma_buf_attachment *attach, + enum dma_data_direction dir) +{ + return attach->dmabuf->ops->map_dma_buf(attach, dir); +} + +static void dma_buf_sgt_compat_unmap_dma_buf(struct dma_buf_attachment *attach, + struct sg_table *sgt, + enum dma_data_direction dir) +{ + attach->dmabuf->ops->unmap_dma_buf(attach, sgt, dir); +} + +/* Route the classic map/unmap ops through the exp ops for old importers */ +static const struct dma_buf_mapping_sgt_exp_ops dma_buf_sgt_compat_exp_ops = { + .map_dma_buf = dma_buf_sgt_compat_map_dma_buf, + .unmap_dma_buf = dma_buf_sgt_compat_unmap_dma_buf, +}; + +/* + * This mapping type is used for unaware exporters that do not support + * match_mapping(). It wraps the dma_buf ops for SGT mappings into a mapping + * type so aware importers can transparently work with unaware exporters. This + * does not require p2p because old exporters will check it through the + * attach->peer2peer mechanism. + */ +const struct dma_buf_mapping_match dma_buf_sgt_exp_compat_match = + DMA_BUF_EMAPPING_SGT(&dma_buf_sgt_compat_exp_ops); diff --git a/include/linux/dma-buf-mapping.h b/include/linux/dma-buf-mapping.h index 28b3a65c0672..69073d589e54 100644 --- a/include/linux/dma-buf-mapping.h +++ b/include/linux/dma-buf-mapping.h @@ -12,6 +12,24 @@ struct dma_buf; struct dma_buf_attachment; struct dma_buf_mapping_exp_ops; +enum dma_sgt_requires_p2p { + DMA_SGT_NO_P2P = 0, + /* + * Exporter: MMIO, importer must accept P2P and pci_p2pdma_distance() + * from exporting_p2p_device must succeed at match time. The match runs + * without the dma_resv lock whenever an importer attaches, so only use + * this if exporting_p2p_device stays bound to its driver for as long as + * the dma_buf exists. Importers that are not PCI devices never match. + */ + DMA_SGT_EXPORTER_REQUIRES_P2P_DISTANCE, + /* + * Exporter: MMIO, importer must accept P2P. No distance check at match; + * the exporter's map does its own, e.g. via dma_buf_phys_vec_to_sgt(). + */ + DMA_SGT_EXPORTER_REQUIRES_P2P, + DMA_SGT_IMPORTER_ACCEPTS_P2P, +}; + /* Type tag for all mapping operations */ struct dma_buf_mapping_exp_ops {}; @@ -109,4 +127,113 @@ dma_buf_attach_map_type(const struct dma_buf_attachment *attach) return &attach->matches[0]; } +/* + * DMA Mapped Scatterlist Type + * + * When this type is matched the map/unmap functions are: + * + * dma_buf_map_sgt() + * dma_buf_unmap_sgt() + * + * The struct sg_table returned by those functions has only the DMA portions + * available. The caller must not try to use the struct page * information. + * + * importing_dma_device is the device passed to dma_buf_dynamic_attach(), + * recorded at match time, and is passed to the DMA API to provide the + * dma_addr_t's. + */ +extern struct dma_buf_mapping_type dma_buf_mapping_sgt_type; + +struct dma_buf_mapping_sgt_exp_ops { + struct dma_buf_mapping_exp_ops ops; + struct sg_table *(*map_dma_buf)(struct dma_buf_attachment *attach, + enum dma_data_direction dir); + void (*unmap_dma_buf)(struct dma_buf_attachment *attach, + struct sg_table *sgt, + enum dma_data_direction dir); +}; + +/** + * dma_buf_sgt_dma_device - Return the device to use for DMA mapping + * @attach: sgt mapping type attachment + * + * Called by the exporter to get the struct device to pass to the DMA API + * during map and unmap callbacks. + */ +static inline struct device * +dma_buf_sgt_dma_device(struct dma_buf_attachment *attach) +{ + if (dma_buf_attach_map_type(attach)->type != &dma_buf_mapping_sgt_type) + return NULL; + return dma_buf_attach_map_type(attach)->sgt_data.importing_dma_device; +} + +/** + * dma_buf_sgt_p2p_allowed - True if MMIO memory can be used peer to peer + * @attach: sgt mapping type attachment + * + * Should be called by exporters, returns true if the exporter's + * DMA_SGT_EXPORTER_REQUIRES_P2P_DISTANCE was matched. + */ +static inline bool dma_buf_sgt_p2p_allowed(struct dma_buf_attachment *attach) +{ + if (dma_buf_attach_map_type(attach)->type != &dma_buf_mapping_sgt_type) + return false; + return dma_buf_attach_map_type(attach)->sgt_data.exporter_requires_p2p == + DMA_SGT_EXPORTER_REQUIRES_P2P_DISTANCE; +} + +static inline const struct dma_buf_mapping_sgt_exp_ops * +dma_buf_get_sgt_ops(struct dma_buf_attachment *attach) +{ + if (dma_buf_attach_map_type(attach)->type != &dma_buf_mapping_sgt_type) + return NULL; + return container_of(dma_buf_attach_map_type(attach)->exp_ops, + struct dma_buf_mapping_sgt_exp_ops, ops); +} + +/* + * Importer SGT entry. Usable in a static table: the importing device is + * supplied by dma_buf_dynamic_attach() and recorded in the matched entry. + */ +#define DMA_BUF_IMAPPING_SGT(_importer_accepts_p2p) \ + ((struct dma_buf_mapping_match){ \ + .type = &dma_buf_mapping_sgt_type, \ + .sgt_data = { .importer_accepts_p2p = (_importer_accepts_p2p) } }) +#define DMA_BUF_EMAPPING_SGT(_exp_ops) \ + ((struct dma_buf_mapping_match){ .type = &dma_buf_mapping_sgt_type, \ + .exp_ops = &((_exp_ops)->ops) }) + +/* + * Only matches if the importing device is P2P capable and the P2P subsystem + * says P2P is possible from p2p_device. + */ +static inline struct dma_buf_mapping_match +DMA_BUF_EMAPPING_SGT_P2P(const struct dma_buf_mapping_sgt_exp_ops *exp_ops, + struct pci_dev *p2p_device) +{ + struct dma_buf_mapping_match match = DMA_BUF_EMAPPING_SGT(exp_ops); + + match.sgt_data.exporter_requires_p2p = + DMA_SGT_EXPORTER_REQUIRES_P2P_DISTANCE; + match.sgt_data.exporting_p2p_device = p2p_device; + return match; +} + +/* + * Only matches if the importer accepts P2P. Whether the importing device can + * actually reach the exporter's MMIO is left to the exporter's map, for + * exporters whose importers need not be PCI devices. + */ +static inline struct dma_buf_mapping_match +DMA_BUF_EMAPPING_SGT_REQUIRES_P2P(const struct dma_buf_mapping_sgt_exp_ops *exp_ops) +{ + struct dma_buf_mapping_match match = DMA_BUF_EMAPPING_SGT(exp_ops); + + match.sgt_data.exporter_requires_p2p = DMA_SGT_EXPORTER_REQUIRES_P2P; + return match; +} + +extern const struct dma_buf_mapping_match dma_buf_sgt_exp_compat_match; + #endif diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h index 81bbcd420d60..de73f931a2f8 100644 --- a/include/linux/dma-buf.h +++ b/include/linux/dma-buf.h @@ -30,6 +30,7 @@ struct dma_buf_attachment; struct dma_buf_mapping_type; struct dma_buf_mapping_exp_ops; + /* * Upper bound on the number of mapping types an attachment can record as * mutually acceptable. An exporter offering more compatible types than this @@ -46,7 +47,23 @@ struct dma_buf_mapping_match { const struct dma_buf_mapping_type *type; const struct dma_buf_mapping_exp_ops *exp_ops; union { - /* Each mapping_type has unique match parameters here */ + /* For dma_buf_mapping_sgt_type */ + struct { + /* Filled from the attaching device, not by importers */ + struct device *importing_dma_device; + /* Only used if DMA_SGT_EXPORTER_REQUIRES_P2P_DISTANCE */ + struct pci_dev *exporting_p2p_device; + /* + * These p2p flags are used to support the hard coded + * mechanism for p2p. If an exporting device knows it + * will put MMIO into the sgt then it should set + * exporter_requires_p2p. Importers should set + * importer_accepts_p2p unless it is known that the + * importing HW never supports P2P because of HW issues. + */ + u8 importer_accepts_p2p; + u8 exporter_requires_p2p; + } sgt_data; }; }; -- 2.54.0
