RE: [PATCH v4 08/25] ocxl: Emit a log message showing how much LPC memory was detected
> -Original Message- > From: Dan Williams > Sent: Wednesday, 1 April 2020 7:49 PM > To: Alastair D'Silva > Cc: Aneesh Kumar K . V ; Oliver O'Halloran > ; Benjamin Herrenschmidt > ; Paul Mackerras ; Michael > Ellerman ; Frederic Barrat ; > Andrew Donnellan ; Arnd Bergmann > ; Greg Kroah-Hartman ; > Vishal Verma ; Dave Jiang > ; Ira Weiny ; Andrew Morton > ; Mauro Carvalho Chehab > ; David S. Miller ; > Rob Herring ; Anton Blanchard ; > Krzysztof Kozlowski ; Mahesh Salgaonkar > ; Madhavan Srinivasan > ; Cédric Le Goater ; Anju T > Sudhakar ; Hari Bathini > ; Thomas Gleixner ; Greg > Kurz ; Nicholas Piggin ; Masahiro > Yamada ; Alexey Kardashevskiy > ; Linux Kernel Mailing List ; > linuxppc-dev ; linux-nvdimm nvd...@lists.01.org>; Linux MM > Subject: Re: [PATCH v4 08/25] ocxl: Emit a log message showing how much > LPC memory was detected > > On Sun, Mar 29, 2020 at 10:23 PM Alastair D'Silva > wrote: > > > > This patch emits a message showing how much LPC memory & special > > purpose memory was detected on an OCXL device. > > > > Signed-off-by: Alastair D'Silva > > Acked-by: Frederic Barrat > > Acked-by: Andrew Donnellan > > --- > > drivers/misc/ocxl/config.c | 4 > > 1 file changed, 4 insertions(+) > > > > diff --git a/drivers/misc/ocxl/config.c b/drivers/misc/ocxl/config.c > > index a62e3d7db2bf..69cca341d446 100644 > > --- a/drivers/misc/ocxl/config.c > > +++ b/drivers/misc/ocxl/config.c > > @@ -568,6 +568,10 @@ static int read_afu_lpc_memory_info(struct > pci_dev *dev, > > afu->special_purpose_mem_size = > > total_mem_size - lpc_mem_size; > > } > > + > > + dev_info(&dev->dev, "Probed LPC memory of %#llx bytes and special > purpose memory of %#llx bytes\n", > > +afu->lpc_mem_size, afu->special_purpose_mem_size); > > A patch for a single log message is too fine grained for my taste, let's > squash > this into another patch in the series. > I'm not sure there's a great place for it. At a pinch, it could with the previous patch, but they are really different layers. > > + > > return 0; > > } > > > > -- > > 2.24.1 > > > > > -- > This email has been checked for viruses by AVG. > https://www.avg.com ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
RE: [PATCH v4 07/25] ocxl: Add functions to map/unmap LPC memory
> -Original Message-
> From: Dan Williams
> Sent: Wednesday, 1 April 2020 7:49 PM
> To: Alastair D'Silva
> Cc: Aneesh Kumar K . V ; Oliver O'Halloran
> ; Benjamin Herrenschmidt
> ; Paul Mackerras ; Michael
> Ellerman ; Frederic Barrat ;
> Andrew Donnellan ; Arnd Bergmann
> ; Greg Kroah-Hartman ;
> Vishal Verma ; Dave Jiang
> ; Ira Weiny ; Andrew Morton
> ; Mauro Carvalho Chehab
> ; David S. Miller ;
> Rob Herring ; Anton Blanchard ;
> Krzysztof Kozlowski ; Mahesh Salgaonkar
> ; Madhavan Srinivasan
> ; Cédric Le Goater ; Anju T
> Sudhakar ; Hari Bathini
> ; Thomas Gleixner ; Greg
> Kurz ; Nicholas Piggin ; Masahiro
> Yamada ; Alexey Kardashevskiy
> ; Linux Kernel Mailing List ;
> linuxppc-dev ; linux-nvdimm nvd...@lists.01.org>; Linux MM
> Subject: Re: [PATCH v4 07/25] ocxl: Add functions to map/unmap LPC
> memory
>
> On Sun, Mar 29, 2020 at 10:23 PM Alastair D'Silva
> wrote:
> >
> > Add functions to map/unmap LPC memory
> >
>
> "map memory" is an overloaded term. I'm guessing this patch has nothing to
> do with mapping memory in the MMU. Is it updating hardware resource
> decoders to start claiming address space that was allocated previously?
>
It's similar to MMIO - these calls end up setting up a BAR which places the LPC
memory into a physical memory range addressable by the kernel.
> > Signed-off-by: Alastair D'Silva
> > Acked-by: Frederic Barrat
> > ---
> > drivers/misc/ocxl/core.c | 51 +++
> > drivers/misc/ocxl/ocxl_internal.h | 3 ++
> > include/misc/ocxl.h | 21 +
> > 3 files changed, 75 insertions(+)
> >
> > diff --git a/drivers/misc/ocxl/core.c b/drivers/misc/ocxl/core.c index
> > 2531c6cf19a0..75ff14e3882a 100644
> > --- a/drivers/misc/ocxl/core.c
> > +++ b/drivers/misc/ocxl/core.c
> > @@ -210,6 +210,56 @@ static void unmap_mmio_areas(struct ocxl_afu
> *afu)
> > release_fn_bar(afu->fn, afu->config.global_mmio_bar); }
> >
> > +int ocxl_afu_map_lpc_mem(struct ocxl_afu *afu) {
> > + struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
> > +
> > + if ((afu->config.lpc_mem_size + afu-
> >config.special_purpose_mem_size) == 0)
> > + return 0;
> > +
> > + afu->lpc_base_addr = ocxl_link_lpc_map(afu->fn->link, dev);
> > + if (afu->lpc_base_addr == 0)
> > + return -EINVAL;
> > +
> > + if (afu->config.lpc_mem_size > 0) {
> > + afu->lpc_res.start = afu->lpc_base_addr + afu-
> >config.lpc_mem_offset;
> > + afu->lpc_res.end = afu->lpc_res.start +
> > afu->config.lpc_mem_size
> - 1;
> > + }
> > +
> > + if (afu->config.special_purpose_mem_size > 0) {
> > + afu->special_purpose_res.start = afu->lpc_base_addr +
> > +
> > afu->config.special_purpose_mem_offset;
> > + afu->special_purpose_res.end =
> > afu->special_purpose_res.start +
> > +
> > afu->config.special_purpose_mem_size - 1;
> > + }
> > +
> > + return 0;
> > +}
> > +EXPORT_SYMBOL_GPL(ocxl_afu_map_lpc_mem);
> > +
> > +struct resource *ocxl_afu_lpc_mem(struct ocxl_afu *afu) {
> > + return &afu->lpc_res;
> > +}
> > +EXPORT_SYMBOL_GPL(ocxl_afu_lpc_mem);
> > +
> > +static void unmap_lpc_mem(struct ocxl_afu *afu) {
> > + struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
> > +
> > + if (afu->lpc_res.start || afu->special_purpose_res.start) {
> > + void *link = afu->fn->link;
> > +
> > + // only release the link when the the last consumer calls
> > release
> > + ocxl_link_lpc_release(link, dev);
> > +
> > + afu->lpc_res.start = 0;
> > + afu->lpc_res.end = 0;
> > + afu->special_purpose_res.start = 0;
> > + afu->special_purpose_res.end = 0;
> > + }
> > +}
> > +
> > static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct
> > pci_dev *dev) {
> > int rc;
> > @@ -251,6 +301,7 @@ static int configure_afu(struct ocxl_afu *afu, u8
> > afu_idx, struct pci_dev *dev)
> >
> > static void deconfigure_afu(struct ocxl_afu *afu) {
> > + unmap_lpc_mem(afu);
>
RE: [PATCH v4 03/25] powerpc/powernv: Map & release OpenCAPI LPC memory
> -Original Message-
> From: Dan Williams
> Sent: Wednesday, 1 April 2020 7:49 PM
> To: Alastair D'Silva
> Cc: Aneesh Kumar K . V ; Oliver O'Halloran
> ; Benjamin Herrenschmidt
> ; Paul Mackerras ; Michael
> Ellerman ; Frederic Barrat ;
> Andrew Donnellan ; Arnd Bergmann
> ; Greg Kroah-Hartman ;
> Vishal Verma ; Dave Jiang
> ; Ira Weiny ; Andrew Morton
> ; Mauro Carvalho Chehab
> ; David S. Miller ;
> Rob Herring ; Anton Blanchard ;
> Krzysztof Kozlowski ; Mahesh Salgaonkar
> ; Madhavan Srinivasan
> ; Cédric Le Goater ; Anju T
> Sudhakar ; Hari Bathini
> ; Thomas Gleixner ; Greg
> Kurz ; Nicholas Piggin ; Masahiro
> Yamada ; Alexey Kardashevskiy
> ; Linux Kernel Mailing List ;
> linuxppc-dev ; linux-nvdimm nvd...@lists.01.org>; Linux MM
> Subject: Re: [PATCH v4 03/25] powerpc/powernv: Map & release OpenCAPI
> LPC memory
>
> On Sun, Mar 29, 2020 at 10:23 PM Alastair D'Silva
> wrote:
> >
> > This patch adds OPAL calls to powernv so that the OpenCAPI driver can
> > map & release LPC (Lowest Point of Coherency) memory.
> >
> > Signed-off-by: Alastair D'Silva
> > Reviewed-by: Andrew Donnellan
> > ---
> > arch/powerpc/include/asm/pnv-ocxl.h | 2 ++
> > arch/powerpc/platforms/powernv/ocxl.c | 43
> > +++
> > 2 files changed, 45 insertions(+)
> >
> > diff --git a/arch/powerpc/include/asm/pnv-ocxl.h
> > b/arch/powerpc/include/asm/pnv-ocxl.h
> > index 7de82647e761..560a19bb71b7 100644
> > --- a/arch/powerpc/include/asm/pnv-ocxl.h
> > +++ b/arch/powerpc/include/asm/pnv-ocxl.h
> > @@ -32,5 +32,7 @@ extern int
> pnv_ocxl_spa_remove_pe_from_cache(void
> > *platform_data, int pe_handle)
> >
> > extern int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr);
> > extern void pnv_ocxl_free_xive_irq(u32 irq);
> > +u64 pnv_ocxl_platform_lpc_setup(struct pci_dev *pdev, u64 size); void
> > +pnv_ocxl_platform_lpc_release(struct pci_dev *pdev);
> >
> > #endif /* _ASM_PNV_OCXL_H */
> > diff --git a/arch/powerpc/platforms/powernv/ocxl.c
> > b/arch/powerpc/platforms/powernv/ocxl.c
> > index 8c65aacda9c8..f13119a7c026 100644
> > --- a/arch/powerpc/platforms/powernv/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/ocxl.c
> > @@ -475,6 +475,49 @@ void pnv_ocxl_spa_release(void *platform_data)
> }
> > EXPORT_SYMBOL_GPL(pnv_ocxl_spa_release);
> >
> > +u64 pnv_ocxl_platform_lpc_setup(struct pci_dev *pdev, u64 size) {
> > + struct pci_controller *hose = pci_bus_to_host(pdev->bus);
> > + struct pnv_phb *phb = hose->private_data;
>
> Is calling the local variable 'hose' instead of 'host' on purpose?
>
Yes, this follows the convention used in other functions in this file.
> > + u32 bdfn = pci_dev_id(pdev);
> > + __be64 base_addr_be64;
> > + u64 base_addr;
> > + int rc;
> > +
> > + rc = opal_npu_mem_alloc(phb->opal_id, bdfn, size,
> &base_addr_be64);
> > + if (rc) {
> > + dev_warn(&pdev->dev,
> > +"OPAL could not allocate LPC memory, rc=%d\n", rc);
> > + return 0;
> > + }
> > +
> > + base_addr = be64_to_cpu(base_addr_be64);
> > +
> > +#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
>
> With the proposed cleanup in patch2 the ifdef can be elided here.
Ok
>
> > + rc = check_hotplug_memory_addressable(base_addr >> PAGE_SHIFT,
> > + size >> PAGE_SHIFT);
> > + if (rc)
> > + return 0;
>
> Is this an error worth logging if someone is wondering why their device is not
> showing up?
>
Yes, I'll add a message.
>
> > +#endif
> > +
> > + return base_addr;
> > +}
> > +EXPORT_SYMBOL_GPL(pnv_ocxl_platform_lpc_setup);
> > +
> > +void pnv_ocxl_platform_lpc_release(struct pci_dev *pdev) {
> > + struct pci_controller *hose = pci_bus_to_host(pdev->bus);
> > + struct pnv_phb *phb = hose->private_data;
> > + u32 bdfn = pci_dev_id(pdev);
> > + int rc;
> > +
> > + rc = opal_npu_mem_release(phb->opal_id, bdfn);
> > + if (rc)
> > + dev_warn(&pdev->dev,
> > +"OPAL reported rc=%d when releasing LPC
> > +memory\n", rc); }
> EXPORT_SYMBOL_GPL(pnv_ocxl_platform_lpc_release);
> > +
> > int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int
> > pe_handle) {
> > struct spa_data *data = (struct spa_data *) platform_data;
> > --
> > 2.24.1
> >
>
>
> --
> This email has been checked for viruses by AVG.
> https://www.avg.com
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
RE: [PATCH v4 02/25] mm/memory_hotplug: Allow check_hotplug_memory_addressable to be called from drivers
> -Original Message-
> From: Dan Williams
> Sent: Wednesday, 1 April 2020 7:48 PM
> To: Alastair D'Silva
> Cc: Aneesh Kumar K . V ; Oliver O'Halloran
> ; Benjamin Herrenschmidt
> ; Paul Mackerras ; Michael
> Ellerman ; Frederic Barrat ;
> Andrew Donnellan ; Arnd Bergmann
> ; Greg Kroah-Hartman ;
> Vishal Verma ; Dave Jiang
> ; Ira Weiny ; Andrew Morton
> ; Mauro Carvalho Chehab
> ; David S. Miller ;
> Rob Herring ; Anton Blanchard ;
> Krzysztof Kozlowski ; Mahesh Salgaonkar
> ; Madhavan Srinivasan
> ; Cédric Le Goater ; Anju T
> Sudhakar ; Hari Bathini
> ; Thomas Gleixner ; Greg
> Kurz ; Nicholas Piggin ; Masahiro
> Yamada ; Alexey Kardashevskiy
> ; Linux Kernel Mailing List ;
> linuxppc-dev ; linux-nvdimm nvd...@lists.01.org>; Linux MM
> Subject: Re: [PATCH v4 02/25] mm/memory_hotplug: Allow
> check_hotplug_memory_addressable to be called from drivers
>
> On Sun, Mar 29, 2020 at 10:23 PM Alastair D'Silva
> wrote:
> >
> > When setting up OpenCAPI connected persistent memory, the range check
> > may not be performed until quite late (or perhaps not at all, if the
> > user does not establish a DAX device).
> >
> > This patch makes the range check callable so we can perform the check
> > while probing the OpenCAPI Persistent Memory device.
> >
> > Signed-off-by: Alastair D'Silva
> > Reviewed-by: Andrew Donnellan
> > ---
> > include/linux/memory_hotplug.h | 5 +
> > mm/memory_hotplug.c| 4 ++--
> > 2 files changed, 7 insertions(+), 2 deletions(-)
> >
> > diff --git a/include/linux/memory_hotplug.h
> > b/include/linux/memory_hotplug.h index f4d59155f3d4..9a19ae0d7e31
> > 100644
> > --- a/include/linux/memory_hotplug.h
> > +++ b/include/linux/memory_hotplug.h
> > @@ -337,6 +337,11 @@ static inline void __remove_memory(int nid, u64
> > start, u64 size) {} extern void set_zone_contiguous(struct zone
> > *zone); extern void clear_zone_contiguous(struct zone *zone);
> >
> > +#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
> > +int check_hotplug_memory_addressable(unsigned long pfn,
> > +unsigned long nr_pages); #endif
> > +/* CONFIG_MEMORY_HOTPLUG_SPARSE */
>
> Let's move this to include/linux/memory.h with the other
> CONFIG_MEMORY_HOTPLUG_SPARSE declarations, and add a dummy
> implementation for the CONFIG_MEMORY_HOTPLUG_SPARSE=n case.
>
> Also, this patch can be squashed with the next one, no need for it to be
> stand alone.
>
Ok
>
> > +
> > extern void __ref free_area_init_core_hotplug(int nid); extern int
> > __add_memory(int nid, u64 start, u64 size); extern int add_memory(int
> > nid, u64 start, u64 size); diff --git a/mm/memory_hotplug.c
> > b/mm/memory_hotplug.c index 0a54ffac8c68..14945f033594 100644
> > --- a/mm/memory_hotplug.c
> > +++ b/mm/memory_hotplug.c
> > @@ -276,8 +276,8 @@ static int check_pfn_span(unsigned long pfn,
> unsigned long nr_pages,
> > return 0;
> > }
> >
> > -static int check_hotplug_memory_addressable(unsigned long pfn,
> > - unsigned long nr_pages)
> > +int check_hotplug_memory_addressable(unsigned long pfn,
> > +unsigned long nr_pages)
> > {
> > const u64 max_addr = PFN_PHYS(pfn + nr_pages) - 1;
> >
> > --
> > 2.24.1
> >
--
Alastair D'Silva mob: 0423 762 819
skype: alastair_dsilva msn: alast...@d-silva.org
blog: http://alastair.d-silva.orgTwitter: @EvilDeece
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
RE: [PATCH v4 01/25] powerpc/powernv: Add OPAL calls for LPC memory alloc/release
> -Original Message- > From: Dan Williams > Sent: Wednesday, 1 April 2020 7:48 PM > To: Alastair D'Silva > Cc: Aneesh Kumar K . V ; Oliver O'Halloran > ; Benjamin Herrenschmidt > ; Paul Mackerras ; Michael > Ellerman ; Frederic Barrat ; > Andrew Donnellan ; Arnd Bergmann > ; Greg Kroah-Hartman ; > Vishal Verma ; Dave Jiang > ; Ira Weiny ; Andrew Morton > ; Mauro Carvalho Chehab > ; David S. Miller ; > Rob Herring ; Anton Blanchard ; > Krzysztof Kozlowski ; Mahesh Salgaonkar > ; Madhavan Srinivasan > ; Cédric Le Goater ; Anju T > Sudhakar ; Hari Bathini > ; Thomas Gleixner ; Greg > Kurz ; Nicholas Piggin ; Masahiro > Yamada ; Alexey Kardashevskiy > ; Linux Kernel Mailing List ; > linuxppc-dev ; linux-nvdimm nvd...@lists.01.org>; Linux MM > Subject: Re: [PATCH v4 01/25] powerpc/powernv: Add OPAL calls for LPC > memory alloc/release > > On Sun, Mar 29, 2020 at 10:23 PM Alastair D'Silva > wrote: > > > > Add OPAL calls for LPC memory alloc/release > > > > This seems to be referencing an existing api definition, can you include a > pointer to the spec in case someone wanted to understand what these > routines do? I suspect this is not allocating memory in the traditional sense > as > much as it's allocating physical address space for a device to be mapped? > These API calls were introduced in the following skiboot commit: https://github.com/open-power/skiboot/commit/1a548857ce1f02f43585b326a891eed18a7b43b3 I'll add it to the description. > > > Signed-off-by: Alastair D'Silva > > Acked-by: Andrew Donnellan > > Acked-by: Frederic Barrat > > --- > > arch/powerpc/include/asm/opal-api.h| 2 ++ > > arch/powerpc/include/asm/opal.h| 2 ++ > > arch/powerpc/platforms/powernv/opal-call.c | 2 ++ > > 3 files changed, 6 insertions(+) > > > > diff --git a/arch/powerpc/include/asm/opal-api.h > > b/arch/powerpc/include/asm/opal-api.h > > index c1f25a760eb1..9298e603001b 100644 > > --- a/arch/powerpc/include/asm/opal-api.h > > +++ b/arch/powerpc/include/asm/opal-api.h > > @@ -208,6 +208,8 @@ > > #define OPAL_HANDLE_HMI2 166 > > #defineOPAL_NX_COPROC_INIT 167 > > #define OPAL_XIVE_GET_VP_STATE 170 > > +#define OPAL_NPU_MEM_ALLOC 171 > > +#define OPAL_NPU_MEM_RELEASE 172 > > #define OPAL_MPIPL_UPDATE 173 > > #define OPAL_MPIPL_REGISTER_TAG174 > > #define OPAL_MPIPL_QUERY_TAG 175 > > diff --git a/arch/powerpc/include/asm/opal.h > > b/arch/powerpc/include/asm/opal.h index 9986ac34b8e2..301fea46c7ca > > 100644 > > --- a/arch/powerpc/include/asm/opal.h > > +++ b/arch/powerpc/include/asm/opal.h > > @@ -39,6 +39,8 @@ int64_t opal_npu_spa_clear_cache(uint64_t phb_id, > uint32_t bdfn, > > uint64_t PE_handle); int64_t > > opal_npu_tl_set(uint64_t phb_id, uint32_t bdfn, long cap, > > uint64_t rate_phys, uint32_t size); > > +int64_t opal_npu_mem_alloc(u64 phb_id, u32 bdfn, u64 size, __be64 > > +*bar); int64_t opal_npu_mem_release(u64 phb_id, u32 bdfn); > > > > int64_t opal_console_write(int64_t term_number, __be64 *length, > >const uint8_t *buffer); diff --git > > a/arch/powerpc/platforms/powernv/opal-call.c > > b/arch/powerpc/platforms/powernv/opal-call.c > > index 5cd0f52d258f..f26e58b72c04 100644 > > --- a/arch/powerpc/platforms/powernv/opal-call.c > > +++ b/arch/powerpc/platforms/powernv/opal-call.c > > @@ -287,6 +287,8 @@ OPAL_CALL(opal_pci_set_pbcq_tunnel_bar, > OPAL_PCI_SET_PBCQ_TUNNEL_BAR); > > OPAL_CALL(opal_sensor_read_u64, > OPAL_SENSOR_READ_U64); > > OPAL_CALL(opal_sensor_group_enable, > OPAL_SENSOR_GROUP_ENABLE); > > OPAL_CALL(opal_nx_coproc_init, OPAL_NX_COPROC_INIT); > > +OPAL_CALL(opal_npu_mem_alloc, OPAL_NPU_MEM_ALLOC); > > +OPAL_CALL(opal_npu_mem_release, > OPAL_NPU_MEM_RELEASE); > > OPAL_CALL(opal_mpipl_update, OPAL_MPIPL_UPDATE); > > OPAL_CALL(opal_mpipl_register_tag, > OPAL_MPIPL_REGISTER_TAG); > > OPAL_CALL(opal_mpipl_query_tag, > OPAL_MPIPL_QUERY_TAG); > > -- > > 2.24.1 > > > > > -- > This email has been checked for viruses by AVG. > https://www.avg.com -- Alastair D'Silva mob: 0423 762 819 skype: alastair_dsilva msn: alast...@d-silva.org blog: http://alastair.d-silva.orgTwitter: @EvilDeece ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
RE: [PATCH v4 00/25] Add support for OpenCAPI Persistent Memory devices
> -Original Message- > From: Dan Williams > Sent: Wednesday, 1 April 2020 7:48 PM > To: Alastair D'Silva > Cc: Aneesh Kumar K . V ; Oliver O'Halloran > ; Benjamin Herrenschmidt > ; Paul Mackerras ; Michael > Ellerman ; Frederic Barrat ; > Andrew Donnellan ; Arnd Bergmann > ; Greg Kroah-Hartman ; > Vishal Verma ; Dave Jiang > ; Ira Weiny ; Andrew Morton > ; Mauro Carvalho Chehab > ; David S. Miller ; > Rob Herring ; Anton Blanchard ; > Krzysztof Kozlowski ; Mahesh Salgaonkar > ; Madhavan Srinivasan > ; Cédric Le Goater ; Anju T > Sudhakar ; Hari Bathini > ; Thomas Gleixner ; Greg > Kurz ; Nicholas Piggin ; Masahiro > Yamada ; Alexey Kardashevskiy > ; Linux Kernel Mailing List ; > linuxppc-dev ; linux-nvdimm nvd...@lists.01.org>; Linux MM > Subject: Re: [PATCH v4 00/25] Add support for OpenCAPI Persistent Memory > devices > > On Sun, Mar 29, 2020 at 10:23 PM Alastair D'Silva > wrote: > > > > This series adds support for OpenCAPI Persistent Memory devices on > > bare metal (arch/powernv), exposing them as nvdimms so that we can > > make use of the existing infrastructure. There already exists a driver > > for the same devices abstracted through PowerVM (arch/pseries): > > arch/powerpc/platforms/pseries/papr_scm.c > > > > These devices are connected via OpenCAPI, and present as LPC (lowest > coherence point) memory to the system, practically, that means that > memory on these cards could be treated as conventional, cache-coherent > memory. > > > > Since the devices are connected via OpenCAPI, they are not enumerated > via ACPI. Instead, OpenCAPI links present as pseudo-PCI bridges, with > devices below them. > > > > This series introduces a driver that exposes the memory on these cards as > nvdimms, with each card getting it's own bus. This is somewhat complicated > by the fact that the cards do not have out of band persistent storage for > metadata, so 1 SECTION_SIZE's (see SPARSEMEM) worth of storage is carved > out of the top of the card storage to implement the ndctl_config_* calls. > > Is it really tied to section-size? Can't that change based on the configured > page-size? It's not clear to me why that would be the choice, but I'll dig > into > the implementation. > I had tried using PAGE_SIZE, but ran into problems carving off just 1 page and handing it to the kernel, while leaving the rest as pmem. That was a while ago though, so maybe I should retry it. > > The driver is not responsible for configuring the NPU (NVLink Processing > Unit) BARs to map the LPC memory from the card into the system's physical > address space, instead, it requests this to be done via OPAL calls (typically > implemented by Skiboot). > > Are OPAL calls similar to ACPI DSMs? I.e. methods for the OS to invoke > platform firmware services? What's Skiboot? > Yes, OPAL is the interface to firmware for POWER. Skiboot is the open-source (and only) implementation of OPAL. > > > > The series is structured as follows: > > - Required infrastructure changes & cleanup > > - A minimal driver implementation > > - Implementing additional features within the driver > > Thanks for the intro and the changelog! > > > > > Changelog: > > V4: > > - Rebase on next-20200320 > > Do you have dependencies on other material that's in -next? Otherwise - > next is only a viable development baseline if you are going to merge through > Andrew's tree. > > > - Bump copyright to 2020 > > - Ensure all uapi headers use C89 compatible comments (missed > ocxlpmem.h) > > - Move the driver back to drivers/nvdimm/ocxl, after confirmation > > that this location is desirable > > - Rename ocxl.c to ocxlpmem.c (+ support files) > > - Rename all ocxl_pmem to ocxlpmem > > - Address checkpatch --strict issues > > - "powerpc/powernv: Add OPAL calls for LPC memory alloc/release" > > - Pass base address as __be64 > > - "ocxl: Tally up the LPC memory on a link & allow it to be mapped" > > - Address checkpatch spacing warnings > > - Reword blurb > > - Reword size description for ocxl_link_add_lpc_mem() > > - Add an early exit in ocxl_link_lpc_release() to avoid triggering > > bogus warnings if called after ocxl_link_lpc_map() fails > > - "powerpc/powernv: Add OPAL calls for LPC memory alloc/release" > > - Reword blurb > > - "powerpc/powernv: Map & release OpenCAPI LPC memory" > > - Reword blurb > > - Move minor_idr
[PATCH v4 17/25] nvdimm/ocxl: Add controller dump IOCTLs
This patch adds IOCTLs to allow userspace to request & fetch dumps
of the internal controller state.
This is useful during debugging or when a fatal error on the controller
has occurred.
The expected flow of operations are:
1. IOCTL_OCXL_PMEM_CONTROLLER_DUMP to request the controller to take
a dump. This IOCTL will complete after the dump is available for
collection.
2. IOCTL_OCXL_PMEM_CONTROLLER_DUMP_DATA called repeatedly to fetch
chunks from the buffer
3. IOCTL_OCXL_PMEM_CONTROLLER_DUMP_COMPLETE to notify the controller
that it can free any internal resources used for the dump
Signed-off-by: Alastair D'Silva
---
drivers/nvdimm/ocxl/main.c | 161 +
include/uapi/nvdimm/ocxlpmem.h | 16
2 files changed, 177 insertions(+)
diff --git a/drivers/nvdimm/ocxl/main.c b/drivers/nvdimm/ocxl/main.c
index e6be0029f658..d0db358ded43 100644
--- a/drivers/nvdimm/ocxl/main.c
+++ b/drivers/nvdimm/ocxl/main.c
@@ -566,6 +566,153 @@ static int ioctl_error_log(struct ocxlpmem *ocxlpmem,
return 0;
}
+/**
+ * controller_dump_header_parse() - Parse the first 64 bits of the controller
dump command response
+ * @ocxlpmem: the device metadata
+ * @length: out, returns the number of bytes in the response (excluding the 64
bit header)
+ */
+static int controller_dump_header_parse(struct ocxlpmem *ocxlpmem, u16 *length)
+{
+ int rc;
+ u64 val;
+ u16 data_identifier;
+ u32 data_length;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ return rc;
+
+ data_identifier = val >> 48;
+ data_length = val & 0x;
+
+ if (data_identifier != 0x4344) { // 'CD'
+ dev_err(&ocxlpmem->dev,
+ "Bad data identifier for error log data, expected 'CD',
got '%2s' (%#x), data_length=%u\n",
+ (char *)&data_identifier,
+ (unsigned int)data_identifier, data_length);
+ return -EINVAL;
+ }
+
+ *length = data_length;
+ return 0;
+}
+
+static int ioctl_controller_dump_data(struct ocxlpmem *ocxlpmem,
+ struct
ioctl_ocxlpmem_controller_dump_data __user *uarg)
+{
+ struct ioctl_ocxlpmem_controller_dump_data args;
+ u64 __user *buf;
+ u16 i, buf_size;
+ u64 val;
+ int rc;
+
+ if (copy_from_user(&args, uarg, sizeof(args)))
+ return -EFAULT;
+
+ if (args.buf_size % sizeof(u64))
+ return -EINVAL;
+
+ if (args.buf_size > ocxlpmem->admin_command.data_size)
+ return -EINVAL;
+
+ buf = (u64 *)args.buf_ptr;
+
+ mutex_lock(&ocxlpmem->admin_command.lock);
+
+ val = ((u64)args.offset) << 32;
+ val |= args.buf_size;
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+ ocxlpmem->admin_command.request_offset +
0x08,
+ OCXL_LITTLE_ENDIAN, val);
+ if (rc)
+ goto out;
+
+ rc = admin_command_execute(ocxlpmem, ADMIN_COMMAND_CONTROLLER_DUMP);
+ if (rc)
+ goto out;
+ if (rc != STATUS_SUCCESS) {
+ warn_status(ocxlpmem,
+ "Unexpected status from controller dump",
+ rc);
+ goto out;
+ }
+
+ rc = controller_dump_header_parse(ocxlpmem, &buf_size);
+ if (rc)
+ goto out;
+
+ buf_size = min((u16)(buf_size + sizeof(u64)), args.buf_size);
+
+ for (i = 0; i < buf_size / sizeof(u64); i++) {
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+
ocxlpmem->admin_command.data_offset +
+ i * sizeof(u64),
+OCXL_HOST_ENDIAN, &val);
+ if (rc)
+ goto out;
+
+ if (copy_to_user(&buf[i], &val, sizeof(u64))) {
+ rc = -EFAULT;
+ goto out;
+ }
+ }
+
+ args.buf_size = buf_size;
+
+ if (copy_to_user(uarg, &args, sizeof(args))) {
+ rc = -EFAULT;
+ goto out;
+ }
+
+ rc = admin_response_handled(ocxlpmem);
+ if (rc)
+ goto out;
+
+out:
+ mutex_unlock(&ocxlpmem->admin_command.lock);
+ return rc;
+}
+
+int request_controller_dump(struct ocxlpmem *ocxlpmem)
+{
+ int rc;
+ u64 busy = 1;
+
+ rc = ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CHIC,
+ OCXL_LITT
[PATCH v4 16/25] nvdimm/ocxl: Implement the Read Error Log command
The read error log command extracts information from the controller's
internal error log.
This patch exposes this information in 2 ways:
- During probe, if an error occurs & a log is available, print it to the
console
- After probe, make the error log available to userspace via an IOCTL.
Userspace is notified of pending error logs in a later patch
("powerpc/powernv/pmem: Forward events to userspace")
Signed-off-by: Alastair D'Silva
---
.../userspace-api/ioctl/ioctl-number.rst | 1 +
drivers/nvdimm/ocxl/main.c| 240 ++
include/uapi/nvdimm/ocxlpmem.h| 46
3 files changed, 287 insertions(+)
create mode 100644 include/uapi/nvdimm/ocxlpmem.h
diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst
b/Documentation/userspace-api/ioctl/ioctl-number.rst
index 9425377615ce..ba0ce7dca643 100644
--- a/Documentation/userspace-api/ioctl/ioctl-number.rst
+++ b/Documentation/userspace-api/ioctl/ioctl-number.rst
@@ -340,6 +340,7 @@ Code Seq#Include File
Comments
0xC0 00-0F linux/usb/iowarrior.h
0xCA 00-0F uapi/misc/cxl.h
0xCA 10-2F uapi/misc/ocxl.h
+0xCA 30-3F uapi/nvdimm/ocxlpmem.h OpenCAPI
Persistent Memory
0xCA 80-BF uapi/scsi/cxlflash_ioctl.h
0xCB 00-1F CBM
serial IEC bus in development:
<mailto:michael.kl...@puffin.lb.shuttle.de>
diff --git a/drivers/nvdimm/ocxl/main.c b/drivers/nvdimm/ocxl/main.c
index 9b85fcd3f1c9..e6be0029f658 100644
--- a/drivers/nvdimm/ocxl/main.c
+++ b/drivers/nvdimm/ocxl/main.c
@@ -13,6 +13,7 @@
#include
#include
#include
+#include
#include "ocxlpmem.h"
static const struct pci_device_id pci_tbl[] = {
@@ -401,10 +402,190 @@ static int file_release(struct inode *inode, struct file
*file)
return 0;
}
+/**
+ * error_log_header_parse() - Parse the first 64 bits of the error log command
response
+ * @ocxlpmem: the device metadata
+ * @length: out, returns the number of bytes in the response (excluding the 64
bit header)
+ */
+static int error_log_header_parse(struct ocxlpmem *ocxlpmem, u16 *length)
+{
+ int rc;
+ u64 val;
+ u16 data_identifier;
+ u32 data_length;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ return rc;
+
+ data_identifier = val >> 48;
+ data_length = val & 0x;
+
+ if (data_identifier != 0x454C) { // 'EL'
+ dev_err(&ocxlpmem->dev,
+ "Bad data identifier for error log data, expected 'EL',
got '%2s' (%#x), data_length=%u\n",
+ (char *)&data_identifier,
+ (unsigned int)data_identifier, data_length);
+ return -EINVAL;
+ }
+
+ *length = data_length;
+ return 0;
+}
+
+static int read_error_log(struct ocxlpmem *ocxlpmem,
+ struct ioctl_ocxlpmem_error_log *log,
+ bool buf_is_user)
+{
+ u64 val;
+ u16 user_buf_length;
+ u16 buf_length;
+ u64 *buf = (u64 *)log->buf_ptr;
+ u16 i;
+ int rc;
+
+ if (log->buf_size % 8)
+ return -EINVAL;
+
+ rc = ocxlpmem_chi(ocxlpmem, &val);
+ if (rc)
+ return rc;
+
+ if (!(val & GLOBAL_MMIO_CHI_ELA))
+ return -EAGAIN;
+
+ user_buf_length = log->buf_size;
+
+ mutex_lock(&ocxlpmem->admin_command.lock);
+
+ rc = admin_command_execute(ocxlpmem, ADMIN_COMMAND_ERRLOG);
+ if (rc != STATUS_SUCCESS) {
+ warn_status(ocxlpmem,
+ "Unexpected status from retrieve error log", rc);
+ goto out;
+ }
+
+ rc = error_log_header_parse(ocxlpmem, &log->buf_size);
+ if (rc)
+ goto out;
+ // log->buf_size now contains the returned buffer size, not the user
size
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset + 0x08,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ goto out;
+
+ log->log_identifier = val >> 32;
+ log->program_reference_code = val & 0x;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset + 0x10,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ goto out;
+
+ log->error_log_typ
[PATCH v4 22/25] nvdimm/ocxl: Add debug IOCTLs
These IOCTLs provide low level access to the card to aid in debugging
controller/FPGA firmware.
Signed-off-by: Alastair D'Silva
---
drivers/nvdimm/ocxl/Kconfig| 6 +
drivers/nvdimm/ocxl/main.c | 198 +
drivers/nvdimm/ocxl/ocxlpmem.h | 2 +-
include/uapi/nvdimm/ocxlpmem.h | 31 ++
4 files changed, 236 insertions(+), 1 deletion(-)
diff --git a/drivers/nvdimm/ocxl/Kconfig b/drivers/nvdimm/ocxl/Kconfig
index c5d927520920..3f44429d70c9 100644
--- a/drivers/nvdimm/ocxl/Kconfig
+++ b/drivers/nvdimm/ocxl/Kconfig
@@ -12,4 +12,10 @@ config OCXL_PMEM
Select N if unsure.
+config OCXL_PMEM_DEBUG
+ bool "OpenCAPI Persistent Memory debugging"
+ depends on OCXL_PMEM
+ help
+ Enables low level IOCTLs for OpenCAPI Persistent Memory firmware
development
+
endif
diff --git a/drivers/nvdimm/ocxl/main.c b/drivers/nvdimm/ocxl/main.c
index 2fbe3f2f77d9..2811bf7efbab 100644
--- a/drivers/nvdimm/ocxl/main.c
+++ b/drivers/nvdimm/ocxl/main.c
@@ -1027,6 +1027,183 @@ int req_controller_health_perf(struct ocxlpmem
*ocxlpmem)
GLOBAL_MMIO_HCI_REQ_HEALTH_PERF);
}
+#ifdef CONFIG_OCXL_PMEM_DEBUG
+/**
+ * enable_fwdebug() - Enable FW debug on the controller
+ * @ocxlpmem: the device metadata
+ * Return: 0 on success, negative on failure
+ */
+static int enable_fwdebug(const struct ocxlpmem *ocxlpmem)
+{
+ return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_HCI,
+ OCXL_LITTLE_ENDIAN,
+ GLOBAL_MMIO_HCI_FW_DEBUG);
+}
+
+/**
+ * disable_fwdebug() - Disable FW debug on the controller
+ * @ocxlpmem: the device metadata
+ * Return: 0 on success, negative on failure
+ */
+static int disable_fwdebug(const struct ocxlpmem *ocxlpmem)
+{
+ return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_HCIC,
+ OCXL_LITTLE_ENDIAN,
+ GLOBAL_MMIO_HCI_FW_DEBUG);
+}
+
+static int ioctl_fwdebug(struct ocxlpmem *ocxlpmem,
+struct ioctl_ocxl_pmem_fwdebug __user *uarg)
+{
+ struct ioctl_ocxl_pmem_fwdebug args;
+ u64 val;
+ int i;
+ int rc;
+
+ if (copy_from_user(&args, uarg, sizeof(args)))
+ return -EFAULT;
+
+ // Buffer size must be a multiple of 8
+ if ((args.buf_size & 0x07))
+ return -EINVAL;
+
+ if (args.buf_size > ocxlpmem->admin_command.data_size)
+ return -EINVAL;
+
+ mutex_lock(&ocxlpmem->admin_command.lock);
+
+ rc = enable_fwdebug(ocxlpmem);
+ if (rc)
+ goto out;
+
+ // Write DebugAction & FunctionCode
+ val = ((u64)args.debug_action << 56) | ((u64)args.function_code << 40);
+
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+ ocxlpmem->admin_command.request_offset +
0x08,
+ OCXL_LITTLE_ENDIAN, val);
+ if (rc)
+ goto out;
+
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+ ocxlpmem->admin_command.request_offset +
0x10,
+ OCXL_LITTLE_ENDIAN,
+ args.debug_parameter_1);
+ if (rc)
+ goto out;
+
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+ ocxlpmem->admin_command.request_offset +
0x18,
+ OCXL_LITTLE_ENDIAN,
+ args.debug_parameter_2);
+ if (rc)
+ goto out;
+
+ // Populate admin command buffer
+ if (args.buf_size) {
+ for (i = 0; i < args.buf_size; i += sizeof(u64)) {
+ u64 val;
+
+ if (copy_from_user(&val, &args.buf[i], sizeof(u64))) {
+ rc = -EFAULT;
+ goto out;
+ }
+
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+
ocxlpmem->admin_command.data_offset + i,
+ OCXL_HOST_ENDIAN, val);
+ if (rc)
+ goto out;
+ }
+ }
+
+ rc = admin_command_execute(ocxlpmem,
+ ocxlpmem->timeouts[ADMIN_COMMAND_FW_DEBUG]);
+ if (rc < 0)
+ goto out;
+ if (rc != STATUS_SUCCESS) {
+ warn_status(ocxlpmem, "Unexpected status from FW Debug", rc);
+ goto out;
+ }
+
+ if (args.buf_size) {
+ for (i = 0; i < args.buf_size; i += sizeof(u6
[PATCH v4 19/25] nvdimm/ocxl: Forward events to userspace
Some of the interrupts that the card generates are better handled
by the userspace daemon, in particular:
Controller Hardware/Firmware Fatal
Controller Dump Available
Error Log available
This patch allows a userspace application to register an eventfd with
the driver via SCM_IOCTL_EVENTFD to receive notifications of these
interrupts.
Userspace can then identify what events have occurred by calling
SCM_IOCTL_EVENT_CHECK and checking against the SCM_IOCTL_EVENT_FOO
masks.
Signed-off-by: Alastair D'Silva
---
drivers/nvdimm/ocxl/main.c | 220 +
drivers/nvdimm/ocxl/ocxlpmem.h | 4 +
include/uapi/nvdimm/ocxlpmem.h | 12 ++
3 files changed, 236 insertions(+)
diff --git a/drivers/nvdimm/ocxl/main.c b/drivers/nvdimm/ocxl/main.c
index 0040fc09cceb..cb6cdc9eb899 100644
--- a/drivers/nvdimm/ocxl/main.c
+++ b/drivers/nvdimm/ocxl/main.c
@@ -10,6 +10,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -301,8 +302,19 @@ static void free_ocxlpmem(struct ocxlpmem *ocxlpmem)
{
int rc;
+ // Disable doorbells
+ (void)ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CHIEC,
+OCXL_LITTLE_ENDIAN,
+GLOBAL_MMIO_CHI_ALL);
+
free_minor(ocxlpmem);
+ if (ocxlpmem->irq_addr[1])
+ iounmap(ocxlpmem->irq_addr[1]);
+
+ if (ocxlpmem->irq_addr[0])
+ iounmap(ocxlpmem->irq_addr[0]);
+
if (ocxlpmem->ocxl_context) {
rc = ocxl_context_detach(ocxlpmem->ocxl_context);
if (rc == -EBUSY)
@@ -398,6 +410,11 @@ static int file_release(struct inode *inode, struct file
*file)
{
struct ocxlpmem *ocxlpmem = file->private_data;
+ if (ocxlpmem->ev_ctx) {
+ eventfd_ctx_put(ocxlpmem->ev_ctx);
+ ocxlpmem->ev_ctx = NULL;
+ }
+
ocxlpmem_put(ocxlpmem);
return 0;
}
@@ -928,6 +945,52 @@ static int ioctl_controller_stats(struct ocxlpmem
*ocxlpmem,
return rc;
}
+static int ioctl_eventfd(struct ocxlpmem *ocxlpmem,
+struct ioctl_ocxlpmem_eventfd __user *uarg)
+{
+ struct ioctl_ocxlpmem_eventfd args;
+
+ if (copy_from_user(&args, uarg, sizeof(args)))
+ return -EFAULT;
+
+ if (ocxlpmem->ev_ctx)
+ return -EBUSY;
+
+ ocxlpmem->ev_ctx = eventfd_ctx_fdget(args.eventfd);
+ if (IS_ERR(ocxlpmem->ev_ctx))
+ return PTR_ERR(ocxlpmem->ev_ctx);
+
+ return 0;
+}
+
+static int ioctl_event_check(struct ocxlpmem *ocxlpmem, u64 __user *uarg)
+{
+ u64 val = 0;
+ int rc;
+ u64 chi = 0;
+
+ rc = ocxlpmem_chi(ocxlpmem, &chi);
+ if (rc < 0)
+ return rc;
+
+ if (chi & GLOBAL_MMIO_CHI_ELA)
+ val |= IOCTL_OCXLPMEM_EVENT_ERROR_LOG_AVAILABLE;
+
+ if (chi & GLOBAL_MMIO_CHI_CDA)
+ val |= IOCTL_OCXLPMEM_EVENT_CONTROLLER_DUMP_AVAILABLE;
+
+ if (chi & GLOBAL_MMIO_CHI_CFFS)
+ val |= IOCTL_OCXLPMEM_EVENT_FIRMWARE_FATAL;
+
+ if (chi & GLOBAL_MMIO_CHI_CHFS)
+ val |= IOCTL_OCXLPMEM_EVENT_HARDWARE_FATAL;
+
+ if (copy_to_user((u64 __user *)uarg, &val, sizeof(val)))
+ return -EFAULT;
+
+ return rc;
+}
+
static long file_ioctl(struct file *file, unsigned int cmd, unsigned long args)
{
struct ocxlpmem *ocxlpmem = file->private_data;
@@ -956,6 +1019,15 @@ static long file_ioctl(struct file *file, unsigned int
cmd, unsigned long args)
rc = ioctl_controller_stats(ocxlpmem,
(struct
ioctl_ocxlpmem_controller_stats __user *)args);
break;
+
+ case IOCTL_OCXLPMEM_EVENTFD:
+ rc = ioctl_eventfd(ocxlpmem,
+ (struct ioctl_ocxlpmem_eventfd __user
*)args);
+ break;
+
+ case IOCTL_OCXLPMEM_EVENT_CHECK:
+ rc = ioctl_event_check(ocxlpmem, (u64 __user *)args);
+ break;
}
return rc;
@@ -1109,6 +1181,148 @@ static void dump_error_log(struct ocxlpmem *ocxlpmem)
kfree(buf);
}
+static irqreturn_t imn0_handler(void *private)
+{
+ struct ocxlpmem *ocxlpmem = private;
+ u64 chi = 0;
+
+ (void)ocxlpmem_chi(ocxlpmem, &chi);
+
+ if (chi & GLOBAL_MMIO_CHI_ELA) {
+ dev_warn(&ocxlpmem->dev, "Error log is available\n");
+
+ if (ocxlpmem->ev_ctx)
+ eventfd_signal(ocxlpmem->ev_ctx, 1);
+ }
+
+ if (chi & GLOBAL_MMIO_CHI_CDA) {
+ dev_warn(&ocxlpmem->dev, "Controller dump is available\n");
+
+ if (ocxlpmem->ev_ctx)
+ eventfd_signal(ocxlpmem->ev_ctx, 1);
+ }
+
+
[PATCH v4 18/25] nvdimm/ocxl: Add an IOCTL to report controller statistics
The controller can report a number of statistics that are useful
in evaluating the performance and reliability of the card.
This patch exposes this information via an IOCTL.
Signed-off-by: Alastair D'Silva
---
drivers/nvdimm/ocxl/main.c | 220 +
include/uapi/nvdimm/ocxlpmem.h | 21
2 files changed, 241 insertions(+)
diff --git a/drivers/nvdimm/ocxl/main.c b/drivers/nvdimm/ocxl/main.c
index d0db358ded43..0040fc09cceb 100644
--- a/drivers/nvdimm/ocxl/main.c
+++ b/drivers/nvdimm/ocxl/main.c
@@ -713,6 +713,221 @@ static int ioctl_controller_dump_complete(struct ocxlpmem
*ocxlpmem)
GLOBAL_MMIO_HCI_CONTROLLER_DUMP_COLLECTED);
}
+/**
+ * controller_stats_header_parse() - Parse the first 64 bits of the controller
stats admin command response
+ * @ocxlpmem: the device metadata
+ * @length: out, returns the number of bytes in the response (excluding the 64
bit header)
+ */
+static int controller_stats_header_parse(struct ocxlpmem *ocxlpmem,
+u32 *length)
+{
+ int rc;
+ u64 val;
+
+ u16 data_identifier;
+ u32 data_length;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ return rc;
+
+ data_identifier = val >> 48;
+ data_length = val & 0x;
+
+ if (data_identifier != 0x4353) { // 'CS'
+ dev_err(&ocxlpmem->dev,
+ "Bad data identifier for error log data, expected 'CS',
got '%2s' (%#x), data_length=%u\n",
+ (char *)&data_identifier,
+ (unsigned int)data_identifier, data_length);
+ return -EINVAL;
+ }
+
+ *length = data_length;
+ return 0;
+}
+
+static int ioctl_controller_stats(struct ocxlpmem *ocxlpmem,
+ struct ioctl_ocxlpmem_controller_stats __user
*uarg)
+{
+ struct ioctl_ocxlpmem_controller_stats args;
+ u32 length;
+ int rc;
+ u64 val;
+
+ memset(&args, '\0', sizeof(args));
+
+ mutex_lock(&ocxlpmem->admin_command.lock);
+
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+ ocxlpmem->admin_command.request_offset +
0x08,
+ OCXL_LITTLE_ENDIAN, 0);
+ if (rc)
+ goto out;
+
+ rc = admin_command_execute(ocxlpmem, ADMIN_COMMAND_CONTROLLER_STATS);
+ if (rc < 0)
+ goto out;
+ if (rc != STATUS_SUCCESS) {
+ warn_status(ocxlpmem,
+ "Unexpected status from controller stats", rc);
+ goto out;
+ }
+
+ rc = controller_stats_header_parse(ocxlpmem, &length);
+ if (rc)
+ goto out;
+
+ if (length != 0x140) // Documented length of controller stats response
+ warn_status(ocxlpmem,
+ "Unexpected length for controller stats data,
expected 0x140, got 0x%x",
+ length);
+
+#define SPID1_OFFSET (ocxlpmem->admin_command.data_offset + 0x08)
+#define SPID2_OFFSET (ocxlpmem->admin_command.data_offset + 0x48)
+#define RESET_INFO (SPID1_OFFSET + 0x08)
+#define UPTIME_LIFE(SPID1_OFFSET + 0x10)
+#define CRITICAL_UTIL (SPID2_OFFSET + 0x08)
+#define HOST_LOAD_COUNT(SPID2_OFFSET + 0x10)
+#define HOST_STORE_COUNT (SPID2_OFFSET + 0x18)
+#define HOST_LOAD_DURATION (SPID2_OFFSET + 0x20)
+#define HOST_STORE_DURATION(SPID2_OFFSET + 0x28)
+#define MEDIA_READ_COUNT (SPID2_OFFSET + 0x50)
+#define MEDIA_WRITE_COUNT (SPID2_OFFSET + 0x58)
+#define MEDIA_READ_DURATION(SPID2_OFFSET + 0x60)
+#define MEDIA_WRITE_DURATION (SPID2_OFFSET + 0x68)
+#define CACHE_READ_HIT_COUNT (SPID2_OFFSET + 0x90)
+#define CACHE_WRITE_HIT_COUNT (SPID2_OFFSET + 0x98)
+#define FAST_WRITE_COUNT (SPID2_OFFSET + 0xA0)
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, SPID1_OFFSET,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ goto out;
+ if ((val >> 56) != 0x01) { // Check the parameter ID
+ rc = -ENODEV;
+ goto out;
+ }
+ if ((val & 0x) != 0x38) { // Check the length
+ rc = -ENODEV;
+ goto out;
+ }
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, RESET_INFO,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ goto out;
+
+ args.reset_count = val >> 32;
+ args.reset_uptime = val & 0x;
+
[PATCH v4 06/25] ocxl: Tally up the LPC memory on a link & allow it to be mapped
OpenCAPI LPC memory is allocated per link, but each link supports
multiple AFUs, and each AFU can have LPC memory assigned to it.
This patch tallys the memory for all AFUs on a link, allowing it
to be mapped in a single operation after the AFUs have been
enumerated.
Signed-off-by: Alastair D'Silva
---
drivers/misc/ocxl/core.c | 10 ++
drivers/misc/ocxl/link.c | 60 +++
drivers/misc/ocxl/ocxl_internal.h | 33 +
3 files changed, 103 insertions(+)
diff --git a/drivers/misc/ocxl/core.c b/drivers/misc/ocxl/core.c
index b7a09b21ab36..2531c6cf19a0 100644
--- a/drivers/misc/ocxl/core.c
+++ b/drivers/misc/ocxl/core.c
@@ -230,8 +230,18 @@ static int configure_afu(struct ocxl_afu *afu, u8 afu_idx,
struct pci_dev *dev)
if (rc)
goto err_free_pasid;
+ if (afu->config.lpc_mem_size || afu->config.special_purpose_mem_size) {
+ rc = ocxl_link_add_lpc_mem(afu->fn->link,
afu->config.lpc_mem_offset,
+ afu->config.lpc_mem_size +
+
afu->config.special_purpose_mem_size);
+ if (rc)
+ goto err_free_mmio;
+ }
+
return 0;
+err_free_mmio:
+ unmap_mmio_areas(afu);
err_free_pasid:
reclaim_afu_pasid(afu);
err_free_actag:
diff --git a/drivers/misc/ocxl/link.c b/drivers/misc/ocxl/link.c
index 58d111afd9f6..af119d3ef79a 100644
--- a/drivers/misc/ocxl/link.c
+++ b/drivers/misc/ocxl/link.c
@@ -84,6 +84,11 @@ struct ocxl_link {
int dev;
atomic_t irq_available;
struct spa *spa;
+ struct mutex lpc_mem_lock; /* protects lpc_mem & lpc_mem_sz */
+ u64 lpc_mem_sz; /* Total amount of LPC memory presented on the link */
+ u64 lpc_mem;
+ int lpc_consumers;
+
void *platform_data;
};
static struct list_head links_list = LIST_HEAD_INIT(links_list);
@@ -396,6 +401,8 @@ static int alloc_link(struct pci_dev *dev, int PE_mask,
struct ocxl_link **out_l
if (rc)
goto err_spa;
+ mutex_init(&link->lpc_mem_lock);
+
/* platform specific hook */
rc = pnv_ocxl_spa_setup(dev, link->spa->spa_mem, PE_mask,
&link->platform_data);
@@ -711,3 +718,56 @@ void ocxl_link_free_irq(void *link_handle, int hw_irq)
atomic_inc(&link->irq_available);
}
EXPORT_SYMBOL_GPL(ocxl_link_free_irq);
+
+int ocxl_link_add_lpc_mem(void *link_handle, u64 offset, u64 size)
+{
+ struct ocxl_link *link = (struct ocxl_link *)link_handle;
+
+ // Check for overflow
+ if (offset > (offset + size))
+ return -EINVAL;
+
+ mutex_lock(&link->lpc_mem_lock);
+ link->lpc_mem_sz = max(link->lpc_mem_sz, offset + size);
+
+ mutex_unlock(&link->lpc_mem_lock);
+
+ return 0;
+}
+
+u64 ocxl_link_lpc_map(void *link_handle, struct pci_dev *pdev)
+{
+ struct ocxl_link *link = (struct ocxl_link *)link_handle;
+
+ mutex_lock(&link->lpc_mem_lock);
+
+ if (!link->lpc_mem)
+ link->lpc_mem = pnv_ocxl_platform_lpc_setup(pdev,
link->lpc_mem_sz);
+
+ if (link->lpc_mem)
+ link->lpc_consumers++;
+ mutex_unlock(&link->lpc_mem_lock);
+
+ return link->lpc_mem;
+}
+
+void ocxl_link_lpc_release(void *link_handle, struct pci_dev *pdev)
+{
+ struct ocxl_link *link = (struct ocxl_link *)link_handle;
+
+ mutex_lock(&link->lpc_mem_lock);
+
+ if (!link->lpc_mem) {
+ mutex_unlock(&link->lpc_mem_lock);
+ return;
+ }
+
+ WARN_ON(--link->lpc_consumers < 0);
+
+ if (link->lpc_consumers == 0) {
+ pnv_ocxl_platform_lpc_release(pdev);
+ link->lpc_mem = 0;
+ }
+
+ mutex_unlock(&link->lpc_mem_lock);
+}
diff --git a/drivers/misc/ocxl/ocxl_internal.h
b/drivers/misc/ocxl/ocxl_internal.h
index 198e4e4bc51d..2d7575225bd7 100644
--- a/drivers/misc/ocxl/ocxl_internal.h
+++ b/drivers/misc/ocxl/ocxl_internal.h
@@ -142,4 +142,37 @@ int ocxl_irq_offset_to_id(struct ocxl_context *ctx, u64
offset);
u64 ocxl_irq_id_to_offset(struct ocxl_context *ctx, int irq_id);
void ocxl_afu_irq_free_all(struct ocxl_context *ctx);
+/**
+ * ocxl_link_add_lpc_mem() - Increment the amount of memory required by an
OpenCAPI link
+ *
+ * @link_handle: The OpenCAPI link handle
+ * @offset: The offset of the memory to add
+ * @size: The number of bytes to increment memory on the link by
+ *
+ * Returns 0 on success, -EINVAL on overflow
+ */
+int ocxl_link_add_lpc_mem(void *link_handle, u64 offset, u64 size);
+
+/**
+ * ocxl_link_lpc_map() - Map the LPC memory for an OpenCAPI device
+ * Since LPC memory belongs to a link, the whole LPC memory available
+ * on the link must be mapped in order to make it
[PATCH v4 15/25] nvdimm/ocxl: Register a character device for userspace to interact with
This patch introduces a character device (/dev/ocxlpmemX) which further
patches will use to interact with userspace, such as error logs,
controller stats and card debug functionality.
Signed-off-by: Alastair D'Silva
---
drivers/nvdimm/ocxl/main.c | 117 -
drivers/nvdimm/ocxl/ocxlpmem.h | 2 +
2 files changed, 117 insertions(+), 2 deletions(-)
diff --git a/drivers/nvdimm/ocxl/main.c b/drivers/nvdimm/ocxl/main.c
index 8db573036423..9b85fcd3f1c9 100644
--- a/drivers/nvdimm/ocxl/main.c
+++ b/drivers/nvdimm/ocxl/main.c
@@ -10,6 +10,7 @@
#include
#include
#include
+#include
#include
#include
#include "ocxlpmem.h"
@@ -356,6 +357,67 @@ static int ocxlpmem_register(struct ocxlpmem *ocxlpmem)
return device_register(&ocxlpmem->dev);
}
+static void ocxlpmem_put(struct ocxlpmem *ocxlpmem)
+{
+ put_device(&ocxlpmem->dev);
+}
+
+static struct ocxlpmem *ocxlpmem_get(struct ocxlpmem *ocxlpmem)
+{
+ return (!get_device(&ocxlpmem->dev)) ? NULL : ocxlpmem;
+}
+
+static struct ocxlpmem *find_and_get_ocxlpmem(dev_t devno)
+{
+ struct ocxlpmem *ocxlpmem;
+ int minor = MINOR(devno);
+
+ mutex_lock(&minors_idr_lock);
+ ocxlpmem = idr_find(&minors_idr, minor);
+ if (ocxlpmem)
+ ocxlpmem_get(ocxlpmem);
+ mutex_unlock(&minors_idr_lock);
+
+ return ocxlpmem;
+}
+
+static int file_open(struct inode *inode, struct file *file)
+{
+ struct ocxlpmem *ocxlpmem;
+
+ ocxlpmem = find_and_get_ocxlpmem(inode->i_rdev);
+ if (!ocxlpmem)
+ return -ENODEV;
+
+ file->private_data = ocxlpmem;
+ return 0;
+}
+
+static int file_release(struct inode *inode, struct file *file)
+{
+ struct ocxlpmem *ocxlpmem = file->private_data;
+
+ ocxlpmem_put(ocxlpmem);
+ return 0;
+}
+
+static const struct file_operations fops = {
+ .owner = THIS_MODULE,
+ .open = file_open,
+ .release= file_release,
+};
+
+/**
+ * create_cdev() - Create the chardev in /dev for the device
+ * @ocxlpmem: the SCM metadata
+ * Return: 0 on success, negative on failure
+ */
+static int create_cdev(struct ocxlpmem *ocxlpmem)
+{
+ cdev_init(&ocxlpmem->cdev, &fops);
+ return cdev_add(&ocxlpmem->cdev, ocxlpmem->dev.devt, 1);
+}
+
/**
* ocxlpmem_remove() - Free an OpenCAPI persistent memory device
* @pdev: the PCI device information struct
@@ -376,6 +438,13 @@ static void remove(struct pci_dev *pdev)
if (ocxlpmem->nvdimm_bus)
nvdimm_bus_unregister(ocxlpmem->nvdimm_bus);
+ /*
+* Remove the cdev early to prevent a race against userspace
+* via the char dev
+*/
+ if (ocxlpmem->cdev.owner)
+ cdev_del(&ocxlpmem->cdev);
+
device_unregister(&ocxlpmem->dev);
}
}
@@ -527,11 +596,18 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
goto err;
}
- if (setup_command_metadata(ocxlpmem)) {
+ rc = setup_command_metadata(ocxlpmem);
+ if (rc) {
dev_err(&pdev->dev, "Could not read command metadata\n");
goto err;
}
+ rc = create_cdev(ocxlpmem);
+ if (rc) {
+ dev_err(&pdev->dev, "Could not create character device\n");
+ goto err;
+ }
+
elapsed = 0;
timeout = ocxlpmem->readiness_timeout +
ocxlpmem->memory_available_timeout;
@@ -599,6 +675,36 @@ static struct pci_driver pci_driver = {
.shutdown = remove,
};
+static int file_init(void)
+{
+ int rc;
+
+ rc = alloc_chrdev_region(&ocxlpmem_dev, 0, NUM_MINORS, "ocxlpmem");
+ if (rc) {
+ idr_destroy(&minors_idr);
+ pr_err("Unable to allocate OpenCAPI persistent memory major
number: %d\n",
+ rc);
+ return rc;
+ }
+
+ ocxlpmem_class = class_create(THIS_MODULE, "ocxlpmem");
+ if (IS_ERR(ocxlpmem_class)) {
+ idr_destroy(&minors_idr);
+ pr_err("Unable to create ocxlpmem class\n");
+ unregister_chrdev_region(ocxlpmem_dev, NUM_MINORS);
+ return PTR_ERR(ocxlpmem_class);
+ }
+
+ return 0;
+}
+
+static void file_exit(void)
+{
+ class_destroy(ocxlpmem_class);
+ unregister_chrdev_region(ocxlpmem_dev, NUM_MINORS);
+ idr_destroy(&minors_idr);
+}
+
static int __init ocxlpmem_init(void)
{
int rc;
@@ -606,16 +712,23 @@ static int __init ocxlpmem_init(void)
mutex_init(&minors_idr_lock);
idr_init(&minors_idr);
- rc = pci_register_driver
[PATCH v4 12/25] nvdimm/ocxl: Add register addresses & status values to the header
These values have been taken from the device specifications.
Signed-off-by: Alastair D'Silva
---
drivers/nvdimm/ocxl/ocxlpmem.h | 73 ++
1 file changed, 73 insertions(+)
diff --git a/drivers/nvdimm/ocxl/ocxlpmem.h b/drivers/nvdimm/ocxl/ocxlpmem.h
index 03fe7a264281..322387873b4b 100644
--- a/drivers/nvdimm/ocxl/ocxlpmem.h
+++ b/drivers/nvdimm/ocxl/ocxlpmem.h
@@ -8,6 +8,79 @@
#define LABEL_AREA_SIZEBIT_ULL(PA_SECTION_SHIFT)
+#define GLOBAL_MMIO_CHI0x000
+#define GLOBAL_MMIO_CHIC 0x008
+#define GLOBAL_MMIO_CHIE 0x010
+#define GLOBAL_MMIO_CHIEC 0x018
+#define GLOBAL_MMIO_HCI0x020
+#define GLOBAL_MMIO_HCIC 0x028
+#define GLOBAL_MMIO_IMA0_OHP 0x040
+#define GLOBAL_MMIO_IMA0_CFP 0x048
+#define GLOBAL_MMIO_IMA1_OHP 0x050
+#define GLOBAL_MMIO_IMA1_CFP 0x058
+#define GLOBAL_MMIO_ACMA_CREQO 0x100
+#define GLOBAL_MMIO_ACMA_CRSPO 0x104
+#define GLOBAL_MMIO_ACMA_CDBO 0x108
+#define GLOBAL_MMIO_ACMA_CDBS 0x10c
+#define GLOBAL_MMIO_NSCMA_CREQO0x120
+#define GLOBAL_MMIO_NSCMA_CRSPO0x124
+#define GLOBAL_MMIO_NSCMA_CDBO 0x128
+#define GLOBAL_MMIO_NSCMA_CDBS 0x12c
+#define GLOBAL_MMIO_CSTS 0x140
+#define GLOBAL_MMIO_FWVER 0x148
+#define GLOBAL_MMIO_CCAP0 0x160
+#define GLOBAL_MMIO_CCAP1 0x168
+
+#define GLOBAL_MMIO_CHI_ACRA BIT_ULL(0)
+#define GLOBAL_MMIO_CHI_NSCRA BIT_ULL(1)
+#define GLOBAL_MMIO_CHI_CRDY BIT_ULL(4)
+#define GLOBAL_MMIO_CHI_CFFS BIT_ULL(5)
+#define GLOBAL_MMIO_CHI_MA BIT_ULL(6)
+#define GLOBAL_MMIO_CHI_ELABIT_ULL(7)
+#define GLOBAL_MMIO_CHI_CDABIT_ULL(8)
+#define GLOBAL_MMIO_CHI_CHFS BIT_ULL(9)
+
+#define GLOBAL_MMIO_CHI_ALL(GLOBAL_MMIO_CHI_ACRA | \
+GLOBAL_MMIO_CHI_NSCRA | \
+GLOBAL_MMIO_CHI_CRDY | \
+GLOBAL_MMIO_CHI_CFFS | \
+GLOBAL_MMIO_CHI_MA | \
+GLOBAL_MMIO_CHI_ELA | \
+GLOBAL_MMIO_CHI_CDA | \
+GLOBAL_MMIO_CHI_CHFS)
+
+#define GLOBAL_MMIO_HCI_ACRW BIT_ULL(0) // ACRW
+#define GLOBAL_MMIO_HCI_NSCRW BIT_ULL(1) // NSCRW
+#define GLOBAL_MMIO_HCI_AFU_RESET BIT_ULL(2) // AR
+#define GLOBAL_MMIO_HCI_FW_DEBUG BIT_ULL(3) // FDE
+#define GLOBAL_MMIO_HCI_CONTROLLER_DUMPBIT_ULL(4) // CD
+#define GLOBAL_MMIO_HCI_CONTROLLER_DUMP_COLLECTED BIT_ULL(5) // CDC
+#define GLOBAL_MMIO_HCI_REQ_HEALTH_PERFBIT_ULL(6) //
CHPD
+
+#define ADMIN_COMMAND_HEARTBEAT0x00u
+#define ADMIN_COMMAND_SHUTDOWN 0x01u
+#define ADMIN_COMMAND_FW_UPDATE0x02u
+#define ADMIN_COMMAND_FW_DEBUG 0x03u
+#define ADMIN_COMMAND_ERRLOG 0x04u
+#define ADMIN_COMMAND_SMART0x05u
+#define ADMIN_COMMAND_CONTROLLER_STATS 0x06u
+#define ADMIN_COMMAND_CONTROLLER_DUMP 0x07u
+#define ADMIN_COMMAND_CMD_CAPS 0x08u
+#define ADMIN_COMMAND_MAX 0x08u
+
+#define STATUS_SUCCESS 0x00
+#define STATUS_MEM_UNAVAILABLE 0x20
+#define STATUS_BLOCKED_BG_TASK 0x21
+#define STATUS_BAD_OPCODE 0x50
+#define STATUS_BAD_REQUEST_PARM0x51
+#define STATUS_BAD_DATA_PARM 0x52
+#define STATUS_DEBUG_BLOCKED 0x70
+#define STATUS_FAIL0xFF
+
+#define STATUS_FW_UPDATE_BLOCKED STATUS_BLOCKED_BG_TASK
+#define STATUS_FW_ARG_INVALID STATUS_BAD_REQUEST_PARM
+#define STATUS_FW_INVALID STATUS_BAD_DATA_PARM
+
struct ocxlpmem {
struct device dev;
struct pci_dev *pdev;
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v4 23/25] nvdimm/ocxl: Expose SMART data via ndctl
This patch retrieves proprietary formatted SMART data and makes it
available via ndctl. A later contribution will be made to ndctl to
parse this data.
Signed-off-by: Alastair D'Silva
---
drivers/nvdimm/ocxl/main.c | 113 +
drivers/nvdimm/ocxl/ocxlpmem.h | 18 ++
include/uapi/linux/ndctl.h | 1 +
3 files changed, 132 insertions(+)
diff --git a/drivers/nvdimm/ocxl/main.c b/drivers/nvdimm/ocxl/main.c
index 2811bf7efbab..92b4389e8cbb 100644
--- a/drivers/nvdimm/ocxl/main.c
+++ b/drivers/nvdimm/ocxl/main.c
@@ -82,6 +82,114 @@ static int ndctl_config_size(struct nd_cmd_get_config_size
*command)
return 0;
}
+/**
+ * smart_header_parse() - Parse the first 64 bits of the SMART admin command
response
+ * @ocxlpmem: the device metadata
+ * @length: out, returns the number of bytes in the response (excluding the 64
bit header)
+ */
+static int smart_header_parse(struct ocxlpmem *ocxlpmem, u32 *length)
+{
+ int rc;
+ u64 val;
+
+ u16 data_identifier;
+ u32 data_length;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ return rc;
+
+ data_identifier = val >> 48;
+ data_length = val & 0x;
+
+ if (data_identifier != 0x534D) { // 'SM'
+ dev_err(&ocxlpmem->dev,
+ "Bad data identifier for smart data, expected 'SM', got
'%2s'\n",
+ (char *)&data_identifier);
+ return -EINVAL;
+ }
+
+ *length = data_length;
+ return 0;
+}
+
+static int ndctl_smart(struct ocxlpmem *ocxlpmem, struct nd_cmd_pkg *pkg)
+{
+ u32 length, i;
+ struct nd_ocxl_smart *out;
+ int rc;
+
+ mutex_lock(&ocxlpmem->admin_command.lock);
+
+ rc = admin_command_execute(ocxlpmem, ADMIN_COMMAND_SMART);
+ if (rc < 0)
+ goto out;
+ if (rc != STATUS_SUCCESS) {
+ warn_status(ocxlpmem, "Unexpected status from SMART", rc);
+ goto out;
+ }
+
+ rc = smart_header_parse(ocxlpmem, &length);
+ if (rc)
+ goto out;
+
+ pkg->nd_fw_size = length + offsetof(struct nd_ocxl_smart, attribs);
+
+ length = min(length, (u32)(pkg->nd_size_out -
+offsetof(struct nd_ocxl_smart, attribs))); // bytes
+ out = (struct nd_ocxl_smart *)pkg->nd_payload;
+ // Each SMART attribute is 2 * 64 bits
+ out->count = length / (2 * sizeof(u64)); // attributes
+
+ for (i = 0; i < length; i += sizeof(u64)) {
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+
ocxlpmem->admin_command.data_offset +
+sizeof(u64) + i,
+OCXL_LITTLE_ENDIAN,
+&out->attribs[i / sizeof(u64)]);
+ if (rc)
+ goto out;
+ }
+
+ rc = admin_response_handled(ocxlpmem);
+ if (rc)
+ goto out;
+
+out:
+ mutex_unlock(&ocxlpmem->admin_command.lock);
+ return rc;
+}
+
+static int ndctl_call(struct ocxlpmem *ocxlpmem, void *buf,
+ unsigned int buf_len)
+{
+ struct nd_cmd_pkg *pkg = buf;
+
+ if (buf_len < sizeof(struct nd_cmd_pkg)) {
+ dev_err(&ocxlpmem->dev, "Invalid ND_CALL size=%u\n", buf_len);
+ return -EINVAL;
+ }
+
+ if (pkg->nd_family != NVDIMM_FAMILY_OCXL) {
+ dev_err(&ocxlpmem->dev, "Invalid ND_CALL family=0x%llx\n",
+ pkg->nd_family);
+ return -EINVAL;
+ }
+
+ switch (pkg->nd_command) {
+ case ND_CMD_OCXL_SMART:
+ return ndctl_smart(ocxlpmem, pkg);
+ default:
+ dev_err(&ocxlpmem->dev, "Invalid ND_CALL command=0x%llx\n",
+ pkg->nd_command);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
static int ndctl(struct nvdimm_bus_descriptor *nd_desc,
struct nvdimm *nvdimm,
unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc)
@@ -90,6 +198,10 @@ static int ndctl(struct nvdimm_bus_descriptor *nd_desc,
struct ocxlpmem, bus_desc);
switch (cmd) {
+ case ND_CMD_CALL:
+ *cmd_rc = ndctl_call(ocxlpmem, buf, buf_len);
+ return 0;
+
case ND_CMD_GET_CONFIG_SIZE:
*cmd_rc = ndctl_config_size(buf);
return 0;
@@ -173,6 +285,7 @@ static int register_lpc_mem(
[PATCH v4 24/25] nvdimm/ocxl: Expose the serial number & firmware version in sysfs
This patch exposes the serial number & firmware version in sysfs,
which will be used by ndctl in userspace to help users identify
the device.
Signed-off-by: Alastair D'Silva
---
drivers/nvdimm/ocxl/main.c | 42 --
1 file changed, 40 insertions(+), 2 deletions(-)
diff --git a/drivers/nvdimm/ocxl/main.c b/drivers/nvdimm/ocxl/main.c
index 92b4389e8cbb..1f422f0d51ef 100644
--- a/drivers/nvdimm/ocxl/main.c
+++ b/drivers/nvdimm/ocxl/main.c
@@ -235,6 +235,43 @@ static int reserve_metadata(struct ocxlpmem *ocxlpmem,
return 0;
}
+static ssize_t serial_show(struct device *device, struct device_attribute
*attr,
+ char *buf)
+{
+ struct nvdimm *nvdimm = to_nvdimm(device);
+ struct ocxlpmem *ocxlpmem = nvdimm_provider_data(nvdimm);
+ const struct ocxl_fn_config *fn_config =
ocxl_function_config(ocxlpmem->ocxl_fn);
+
+ return scnprintf(buf, PAGE_SIZE, "%llu\n", fn_config->serial);
+}
+static DEVICE_ATTR_RO(serial);
+
+static ssize_t fw_version_show(struct device *device,
+ struct device_attribute *attr, char *buf)
+{
+ struct nvdimm *nvdimm = to_nvdimm(device);
+ struct ocxlpmem *ocxlpmem = nvdimm_provider_data(nvdimm);
+
+ return scnprintf(buf, PAGE_SIZE, "%s\n", ocxlpmem->fw_version);
+}
+static DEVICE_ATTR_RO(fw_version);
+
+static struct attribute *ocxl_pmem_attrs[] = {
+ &dev_attr_serial.attr,
+ &dev_attr_fw_version.attr,
+ NULL,
+};
+
+static const struct attribute_group ocxl_pmem_attribute_group = {
+ .name = "ocxlpmem",
+ .attrs = ocxl_pmem_attrs,
+};
+
+static const struct attribute_group *ocxl_pmem_dimm_attribute_groups[] = {
+ &ocxl_pmem_attribute_group,
+ NULL,
+};
+
/**
* register_lpc_mem() - Discover persistent memory on a device and register it
with the NVDIMM subsystem
* @ocxlpmem: the device metadata
@@ -291,8 +328,9 @@ static int register_lpc_mem(struct ocxlpmem *ocxlpmem)
snprintf(serial, sizeof(serial), "%llx", fn_config->serial);
nd_mapping_desc.nvdimm = nvdimm_create(ocxlpmem->nvdimm_bus, ocxlpmem,
- NULL, nvdimm_flags,
- nvdimm_cmd_mask, 0, NULL);
+ ocxl_pmem_dimm_attribute_groups,
+ nvdimm_flags, nvdimm_cmd_mask, 0,
+ NULL);
if (!nd_mapping_desc.nvdimm)
return -ENOMEM;
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v4 20/25] nvdimm/ocxl: Add an IOCTL to request controller health & perf data
When health & performance data is requested from the controller,
it responds with an error log containing the requested information.
This patch allows the request to be issued via an IOCTL, the data
can later be collected in userspace via the error log IOCTL introduced
in a previous patch. Userspace will be notified of pending error
logs via an event.
Signed-off-by: Alastair D'Silva
---
drivers/nvdimm/ocxl/main.c | 16
include/uapi/nvdimm/ocxlpmem.h | 1 +
2 files changed, 17 insertions(+)
diff --git a/drivers/nvdimm/ocxl/main.c b/drivers/nvdimm/ocxl/main.c
index cb6cdc9eb899..a4315472683c 100644
--- a/drivers/nvdimm/ocxl/main.c
+++ b/drivers/nvdimm/ocxl/main.c
@@ -991,6 +991,18 @@ static int ioctl_event_check(struct ocxlpmem *ocxlpmem,
u64 __user *uarg)
return rc;
}
+/**
+ * req_controller_health_perf() - Request controller health & performance data
+ * @ocxlpmem: the device metadata
+ * Return: 0 on success, negative on failure
+ */
+int req_controller_health_perf(struct ocxlpmem *ocxlpmem)
+{
+ return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_HCI,
+ OCXL_LITTLE_ENDIAN,
+ GLOBAL_MMIO_HCI_REQ_HEALTH_PERF);
+}
+
static long file_ioctl(struct file *file, unsigned int cmd, unsigned long args)
{
struct ocxlpmem *ocxlpmem = file->private_data;
@@ -1028,6 +1040,10 @@ static long file_ioctl(struct file *file, unsigned int
cmd, unsigned long args)
case IOCTL_OCXLPMEM_EVENT_CHECK:
rc = ioctl_event_check(ocxlpmem, (u64 __user *)args);
break;
+
+ case IOCTL_OCXLPMEM_REQUEST_HEALTH:
+ rc = req_controller_health_perf(ocxlpmem);
+ break;
}
return rc;
diff --git a/include/uapi/nvdimm/ocxlpmem.h b/include/uapi/nvdimm/ocxlpmem.h
index d573bd307e35..9c5c8585c1c2 100644
--- a/include/uapi/nvdimm/ocxlpmem.h
+++ b/include/uapi/nvdimm/ocxlpmem.h
@@ -91,5 +91,6 @@ struct ioctl_ocxlpmem_eventfd {
#define IOCTL_OCXLPMEM_CONTROLLER_STATS
_IO(OCXLPMEM_MAGIC, 0x34)
#define IOCTL_OCXLPMEM_EVENTFD _IOW(OCXLPMEM_MAGIC,
0x35, struct ioctl_ocxlpmem_eventfd)
#define IOCTL_OCXLPMEM_EVENT_CHECK _IOR(OCXLPMEM_MAGIC,
0x36, __u64)
+#define IOCTL_OCXLPMEM_REQUEST_HEALTH _IO(OCXLPMEM_MAGIC,
0x37)
#endif /* _UAPI_OCXL_SCM_H */
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v4 21/25] nvdimm/ocxl: Implement the heartbeat command
The heartbeat admin command is a simple admin command that exercises
the communication mechanisms within the controller.
This patch issues a heartbeat command to the card during init to ensure
we can communicate with the card's controller.
Signed-off-by: Alastair D'Silva
Reviewed-by: Andrew Donnellan
---
drivers/nvdimm/ocxl/main.c | 30 ++
1 file changed, 30 insertions(+)
diff --git a/drivers/nvdimm/ocxl/main.c b/drivers/nvdimm/ocxl/main.c
index a4315472683c..2fbe3f2f77d9 100644
--- a/drivers/nvdimm/ocxl/main.c
+++ b/drivers/nvdimm/ocxl/main.c
@@ -272,6 +272,30 @@ static int setup_command_metadata(struct ocxlpmem
*ocxlpmem)
return 0;
}
+/**
+ * heartbeat() - Issue a heartbeat command to the controller
+ * @ocxlpmem: the device metadata
+ * Return: 0 if the controller responded correctly, negative on error
+ */
+static int heartbeat(struct ocxlpmem *ocxlpmem)
+{
+ int rc;
+
+ mutex_lock(&ocxlpmem->admin_command.lock);
+
+ rc = admin_command_execute(ocxlpmem, ADMIN_COMMAND_HEARTBEAT);
+ if (rc < 0)
+ goto out;
+ if (rc != STATUS_SUCCESS)
+ warn_status(ocxlpmem, "Unexpected status from heartbeat", rc);
+
+ (void)admin_response_handled(ocxlpmem);
+
+out:
+ mutex_unlock(&ocxlpmem->admin_command.lock);
+ return rc;
+}
+
/**
* allocate_minor() - Allocate a minor number to use for an OpenCAPI pmem
device
* @ocxlpmem: the device metadata
@@ -1460,6 +1484,12 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
goto err;
}
+ rc = heartbeat(ocxlpmem);
+ if (rc) {
+ dev_err(&pdev->dev, "Heartbeat failed\n");
+ goto err;
+ }
+
elapsed = 0;
timeout = ocxlpmem->readiness_timeout +
ocxlpmem->memory_available_timeout;
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v4 25/25] MAINTAINERS: Add myself & nvdimm/ocxl to ocxl
The OpenCAPI Persistent Memory driver will be maintained as part ofi the ppc tree. I'm also adding myself as an author of the driver & contributor to the generic ocxl driver. Signed-off-by: Alastair D'Silva --- MAINTAINERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index f8670989ec91..3fb9a9f576a7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12064,13 +12064,16 @@ F:tools/objtool/ OCXL (Open Coherent Accelerator Processor Interface OpenCAPI) DRIVER M: Frederic Barrat M: Andrew Donnellan +M: Alastair D'Silva L: linuxppc-...@lists.ozlabs.org S: Supported F: arch/powerpc/platforms/powernv/ocxl.c +F: arch/powerpc/platforms/powernv/pmem/* F: arch/powerpc/include/asm/pnv-ocxl.h F: drivers/misc/ocxl/ F: include/misc/ocxl* F: include/uapi/misc/ocxl.h +F: include/uapi/nvdimm/ocxl-pmem.h F: Documentation/userspace-api/accelerators/ocxl.rst OMAP AUDIO SUPPORT -- 2.24.1 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v4 13/25] nvdimm/ocxl: Read the capability registers & wait for device ready
This patch reads timeouts & firmware version from the controller, and
uses those timeouts to wait for the controller to report that it is ready
before handing the memory over to libnvdimm.
Signed-off-by: Alastair D'Silva
---
drivers/nvdimm/ocxl/Makefile| 2 +-
drivers/nvdimm/ocxl/main.c | 85 +
drivers/nvdimm/ocxl/ocxlpmem.h | 29 +
drivers/nvdimm/ocxl/ocxlpmem_internal.c | 19 ++
4 files changed, 134 insertions(+), 1 deletion(-)
create mode 100644 drivers/nvdimm/ocxl/ocxlpmem_internal.c
diff --git a/drivers/nvdimm/ocxl/Makefile b/drivers/nvdimm/ocxl/Makefile
index e0e8ade1987a..bab97082e062 100644
--- a/drivers/nvdimm/ocxl/Makefile
+++ b/drivers/nvdimm/ocxl/Makefile
@@ -4,4 +4,4 @@ ccflags-$(CONFIG_PPC_WERROR)+= -Werror
obj-$(CONFIG_OCXL_PMEM) += ocxlpmem.o
-ocxlpmem-y := main.o
\ No newline at end of file
+ocxlpmem-y := main.o ocxlpmem_internal.o
diff --git a/drivers/nvdimm/ocxl/main.c b/drivers/nvdimm/ocxl/main.c
index c0066fedf9cc..be76acd33d74 100644
--- a/drivers/nvdimm/ocxl/main.c
+++ b/drivers/nvdimm/ocxl/main.c
@@ -8,6 +8,7 @@
#include
#include
+#include
#include
#include
#include
@@ -327,6 +328,50 @@ static void remove(struct pci_dev *pdev)
}
}
+/**
+ * read_device_metadata() - Retrieve config information from the AFU and save
it for future use
+ * @ocxlpmem: the device metadata
+ * Return: 0 on success, negative on failure
+ */
+static int read_device_metadata(struct ocxlpmem *ocxlpmem)
+{
+ u64 val;
+ int rc;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CCAP0,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ return rc;
+
+ ocxlpmem->scm_revision = val & 0x;
+ ocxlpmem->read_latency = (val >> 32) & 0x;
+ ocxlpmem->readiness_timeout = (val >> 48) & 0x0F;
+ ocxlpmem->memory_available_timeout = val >> 52;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CCAP1,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ return rc;
+
+ ocxlpmem->max_controller_dump_size = val & 0x;
+
+ // Extract firmware version text
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_FWVER,
+OCXL_HOST_ENDIAN,
+(u64 *)ocxlpmem->fw_version);
+ if (rc)
+ return rc;
+
+ ocxlpmem->fw_version[8] = '\0';
+
+ dev_info(&ocxlpmem->dev,
+"Firmware version '%s' SCM revision %d:%d\n",
+ocxlpmem->fw_version, ocxlpmem->scm_revision >> 4,
+ocxlpmem->scm_revision & 0x0F);
+
+ return 0;
+}
+
/**
* probe_function0() - Set up function 0 for an OpenCAPI persistent memory
device
* This is important as it enables templates higher than 0 across all other
@@ -359,6 +404,9 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
{
struct ocxlpmem *ocxlpmem;
int rc;
+ u64 chi;
+ u16 elapsed, timeout;
+ bool ready = false;
if (PCI_FUNC(pdev->devfn) == 0)
return probe_function0(pdev);
@@ -413,6 +461,43 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
goto err;
}
+ rc = read_device_metadata(ocxlpmem);
+ if (rc) {
+ dev_err(&pdev->dev, "Could not read metadata\n");
+ goto err;
+ }
+
+ elapsed = 0;
+ timeout = ocxlpmem->readiness_timeout +
+ ocxlpmem->memory_available_timeout;
+
+ while (true) {
+ rc = ocxlpmem_chi(ocxlpmem, &chi);
+ ready = (chi & (GLOBAL_MMIO_CHI_CRDY | GLOBAL_MMIO_CHI_MA)) ==
+ (GLOBAL_MMIO_CHI_CRDY | GLOBAL_MMIO_CHI_MA);
+
+ if (ready)
+ break;
+
+ if (elapsed++ > timeout) {
+ dev_err(&ocxlpmem->dev,
+ "OpenCAPI Persistent Memory ready timeout.\n");
+
+ if (!(chi & GLOBAL_MMIO_CHI_CRDY))
+ dev_err(&ocxlpmem->dev,
+ "controller is not ready.\n");
+
+ if (!(chi & GLOBAL_MMIO_CHI_MA))
+ dev_err(&ocxlpmem->dev,
+ "controller does not have memory
available.\n");
+
+ rc = -ENXIO;
+ goto err;
+ }
+
+ msleep(1000);
+ }
+
rc = register_lpc_mem(ocxlpmem);
if (rc) {
dev_err(&
[PATCH v4 04/25] ocxl: Remove unnecessary externs
Function declarations don't need externs, remove the existing ones so they are consistent with newer code Signed-off-by: Alastair D'Silva Acked-by: Andrew Donnellan Acked-by: Frederic Barrat --- arch/powerpc/include/asm/pnv-ocxl.h | 40 ++--- include/misc/ocxl.h | 6 ++--- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/arch/powerpc/include/asm/pnv-ocxl.h b/arch/powerpc/include/asm/pnv-ocxl.h index 560a19bb71b7..205efc41a33c 100644 --- a/arch/powerpc/include/asm/pnv-ocxl.h +++ b/arch/powerpc/include/asm/pnv-ocxl.h @@ -9,29 +9,27 @@ #define PNV_OCXL_TL_BITS_PER_RATE 4 #define PNV_OCXL_TL_RATE_BUF_SIZE ((PNV_OCXL_TL_MAX_TEMPLATE+1) * PNV_OCXL_TL_BITS_PER_RATE / 8) -extern int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled, - u16 *supported); -extern int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count); +int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled, u16 *supported); +int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count); -extern int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap, +int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap, char *rate_buf, int rate_buf_size); -extern int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap, - uint64_t rate_buf_phys, int rate_buf_size); - -extern int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq); -extern void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar, - void __iomem *tfc, void __iomem *pe_handle); -extern int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr, - void __iomem **dar, void __iomem **tfc, - void __iomem **pe_handle); - -extern int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask, - void **platform_data); -extern void pnv_ocxl_spa_release(void *platform_data); -extern int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle); - -extern int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr); -extern void pnv_ocxl_free_xive_irq(u32 irq); +int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap, +uint64_t rate_buf_phys, int rate_buf_size); + +int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq); +void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar, +void __iomem *tfc, void __iomem *pe_handle); +int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr, + void __iomem **dar, void __iomem **tfc, + void __iomem **pe_handle); + +int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask, void **platform_data); +void pnv_ocxl_spa_release(void *platform_data); +int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle); + +int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr); +void pnv_ocxl_free_xive_irq(u32 irq); u64 pnv_ocxl_platform_lpc_setup(struct pci_dev *pdev, u64 size); void pnv_ocxl_platform_lpc_release(struct pci_dev *pdev); diff --git a/include/misc/ocxl.h b/include/misc/ocxl.h index 06dd5839e438..0a762e387418 100644 --- a/include/misc/ocxl.h +++ b/include/misc/ocxl.h @@ -173,7 +173,7 @@ int ocxl_context_detach(struct ocxl_context *ctx); * * Returns 0 on success, negative on failure */ -extern int ocxl_afu_irq_alloc(struct ocxl_context *ctx, int *irq_id); +int ocxl_afu_irq_alloc(struct ocxl_context *ctx, int *irq_id); /** * Frees an IRQ associated with an AFU context @@ -182,7 +182,7 @@ extern int ocxl_afu_irq_alloc(struct ocxl_context *ctx, int *irq_id); * * Returns 0 on success, negative on failure */ -extern int ocxl_afu_irq_free(struct ocxl_context *ctx, int irq_id); +int ocxl_afu_irq_free(struct ocxl_context *ctx, int irq_id); /** * Gets the address of the trigger page for an IRQ @@ -193,7 +193,7 @@ extern int ocxl_afu_irq_free(struct ocxl_context *ctx, int irq_id); * * returns the trigger page address, or 0 if the IRQ is not valid */ -extern u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, int irq_id); +u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, int irq_id); /** * Provide a callback to be called when an IRQ is triggered -- 2.24.1 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v4 10/25] nvdimm: Add driver for OpenCAPI Persistent Memory
This driver exposes LPC memory on OpenCAPI pmem cards
as an NVDIMM, allowing the existing nvram infrastructure
to be used.
Namespace metadata is stored on the media itself, so
scm_reserve_metadata() maps 1 section's worth of PMEM storage
at the start to hold this. The rest of the PMEM range is registered
with libnvdimm as an nvdimm. ndctl_config_read/write/size() provide
callbacks to libnvdimm to access the metadata.
Signed-off-by: Alastair D'Silva
---
drivers/nvdimm/Kconfig | 2 +
drivers/nvdimm/Makefile| 1 +
drivers/nvdimm/ocxl/Kconfig| 15 ++
drivers/nvdimm/ocxl/Makefile | 7 +
drivers/nvdimm/ocxl/main.c | 476 +
drivers/nvdimm/ocxl/ocxlpmem.h | 23 ++
6 files changed, 524 insertions(+)
create mode 100644 drivers/nvdimm/ocxl/Kconfig
create mode 100644 drivers/nvdimm/ocxl/Makefile
create mode 100644 drivers/nvdimm/ocxl/main.c
create mode 100644 drivers/nvdimm/ocxl/ocxlpmem.h
diff --git a/drivers/nvdimm/Kconfig b/drivers/nvdimm/Kconfig
index b7d1eb38b27d..368328637182 100644
--- a/drivers/nvdimm/Kconfig
+++ b/drivers/nvdimm/Kconfig
@@ -131,4 +131,6 @@ config NVDIMM_TEST_BUILD
core devm_memremap_pages() implementation and other
infrastructure.
+source "drivers/nvdimm/ocxl/Kconfig"
+
endif
diff --git a/drivers/nvdimm/Makefile b/drivers/nvdimm/Makefile
index 29203f3d3069..bc02be11c794 100644
--- a/drivers/nvdimm/Makefile
+++ b/drivers/nvdimm/Makefile
@@ -33,3 +33,4 @@ libnvdimm-$(CONFIG_NVDIMM_KEYS) += security.o
TOOLS := ../../tools
TEST_SRC := $(TOOLS)/testing/nvdimm/test
obj-$(CONFIG_NVDIMM_TEST_BUILD) += $(TEST_SRC)/iomap.o
+obj-$(CONFIG_LIBNVDIMM) += ocxl/
diff --git a/drivers/nvdimm/ocxl/Kconfig b/drivers/nvdimm/ocxl/Kconfig
new file mode 100644
index ..c5d927520920
--- /dev/null
+++ b/drivers/nvdimm/ocxl/Kconfig
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0-only
+if LIBNVDIMM
+
+config OCXL_PMEM
+ tristate "OpenCAPI Persistent Memory"
+ depends on LIBNVDIMM && PPC_POWERNV && PCI && EEH && ZONE_DEVICE && OCXL
+ help
+ Exposes devices that implement the OpenCAPI Storage Class Memory
+ specification as persistent memory regions. You may also want
+ DEV_DAX, DEV_DAX_PMEM & FS_DAX if you plan on using DAX devices
+ stacked on top of this driver.
+
+ Select N if unsure.
+
+endif
diff --git a/drivers/nvdimm/ocxl/Makefile b/drivers/nvdimm/ocxl/Makefile
new file mode 100644
index ..e0e8ade1987a
--- /dev/null
+++ b/drivers/nvdimm/ocxl/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+
+ccflags-$(CONFIG_PPC_WERROR) += -Werror
+
+obj-$(CONFIG_OCXL_PMEM) += ocxlpmem.o
+
+ocxlpmem-y := main.o
\ No newline at end of file
diff --git a/drivers/nvdimm/ocxl/main.c b/drivers/nvdimm/ocxl/main.c
new file mode 100644
index ..c0066fedf9cc
--- /dev/null
+++ b/drivers/nvdimm/ocxl/main.c
@@ -0,0 +1,476 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2020 IBM Corp.
+
+/*
+ * A driver for OpenCAPI devices that implement the Storage Class
+ * Memory specification.
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include "ocxlpmem.h"
+
+static const struct pci_device_id pci_tbl[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0625), },
+ { }
+};
+
+MODULE_DEVICE_TABLE(pci, pci_tbl);
+
+#define NUM_MINORS 256 // Total to reserve
+
+static dev_t ocxlpmem_dev;
+static struct class *ocxlpmem_class;
+static struct mutex minors_idr_lock;
+static struct idr minors_idr;
+
+/**
+ * ndctl_config_write() - Handle a ND_CMD_SET_CONFIG_DATA command from ndctl
+ * @ocxlpmem: the device metadata
+ * @command: the incoming data to write
+ * Return: 0 on success, negative on failure
+ */
+static int ndctl_config_write(struct ocxlpmem *ocxlpmem,
+ struct nd_cmd_set_config_hdr *command)
+{
+ if (command->in_offset + command->in_length > LABEL_AREA_SIZE)
+ return -EINVAL;
+
+ memcpy_flushcache(ocxlpmem->metadata_addr + command->in_offset,
+ command->in_buf, command->in_length);
+
+ return 0;
+}
+
+/**
+ * ndctl_config_read() - Handle a ND_CMD_GET_CONFIG_DATA command from ndctl
+ * @ocxlpmem: the device metadata
+ * @command: the read request
+ * Return: 0 on success, negative on failure
+ */
+static int ndctl_config_read(struct ocxlpmem *ocxlpmem,
+struct nd_cmd_get_config_data_hdr *command)
+{
+ if (command->in_offset + command->in_length > LABEL_AREA_SIZE)
+ return -EINVAL;
+
+ memcpy_mcsafe(command->out_buf,
+ ocxlpmem->metadata_addr + command->in_offset,
+ command->in_length);
+
+ return 0;
+}
+
+/**
+ * ndctl_config_size() - Handle a ND_CMD_GET_CONFIG_SIZE command from ndctl
+ * @command: the read r
[PATCH v4 03/25] powerpc/powernv: Map & release OpenCAPI LPC memory
This patch adds OPAL calls to powernv so that the OpenCAPI
driver can map & release LPC (Lowest Point of Coherency) memory.
Signed-off-by: Alastair D'Silva
Reviewed-by: Andrew Donnellan
---
arch/powerpc/include/asm/pnv-ocxl.h | 2 ++
arch/powerpc/platforms/powernv/ocxl.c | 43 +++
2 files changed, 45 insertions(+)
diff --git a/arch/powerpc/include/asm/pnv-ocxl.h
b/arch/powerpc/include/asm/pnv-ocxl.h
index 7de82647e761..560a19bb71b7 100644
--- a/arch/powerpc/include/asm/pnv-ocxl.h
+++ b/arch/powerpc/include/asm/pnv-ocxl.h
@@ -32,5 +32,7 @@ extern int pnv_ocxl_spa_remove_pe_from_cache(void
*platform_data, int pe_handle)
extern int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr);
extern void pnv_ocxl_free_xive_irq(u32 irq);
+u64 pnv_ocxl_platform_lpc_setup(struct pci_dev *pdev, u64 size);
+void pnv_ocxl_platform_lpc_release(struct pci_dev *pdev);
#endif /* _ASM_PNV_OCXL_H */
diff --git a/arch/powerpc/platforms/powernv/ocxl.c
b/arch/powerpc/platforms/powernv/ocxl.c
index 8c65aacda9c8..f13119a7c026 100644
--- a/arch/powerpc/platforms/powernv/ocxl.c
+++ b/arch/powerpc/platforms/powernv/ocxl.c
@@ -475,6 +475,49 @@ void pnv_ocxl_spa_release(void *platform_data)
}
EXPORT_SYMBOL_GPL(pnv_ocxl_spa_release);
+u64 pnv_ocxl_platform_lpc_setup(struct pci_dev *pdev, u64 size)
+{
+ struct pci_controller *hose = pci_bus_to_host(pdev->bus);
+ struct pnv_phb *phb = hose->private_data;
+ u32 bdfn = pci_dev_id(pdev);
+ __be64 base_addr_be64;
+ u64 base_addr;
+ int rc;
+
+ rc = opal_npu_mem_alloc(phb->opal_id, bdfn, size, &base_addr_be64);
+ if (rc) {
+ dev_warn(&pdev->dev,
+"OPAL could not allocate LPC memory, rc=%d\n", rc);
+ return 0;
+ }
+
+ base_addr = be64_to_cpu(base_addr_be64);
+
+#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
+ rc = check_hotplug_memory_addressable(base_addr >> PAGE_SHIFT,
+ size >> PAGE_SHIFT);
+ if (rc)
+ return 0;
+#endif
+
+ return base_addr;
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_platform_lpc_setup);
+
+void pnv_ocxl_platform_lpc_release(struct pci_dev *pdev)
+{
+ struct pci_controller *hose = pci_bus_to_host(pdev->bus);
+ struct pnv_phb *phb = hose->private_data;
+ u32 bdfn = pci_dev_id(pdev);
+ int rc;
+
+ rc = opal_npu_mem_release(phb->opal_id, bdfn);
+ if (rc)
+ dev_warn(&pdev->dev,
+"OPAL reported rc=%d when releasing LPC memory\n", rc);
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_platform_lpc_release);
+
int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle)
{
struct spa_data *data = (struct spa_data *) platform_data;
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v4 08/25] ocxl: Emit a log message showing how much LPC memory was detected
This patch emits a message showing how much LPC memory & special purpose memory was detected on an OCXL device. Signed-off-by: Alastair D'Silva Acked-by: Frederic Barrat Acked-by: Andrew Donnellan --- drivers/misc/ocxl/config.c | 4 1 file changed, 4 insertions(+) diff --git a/drivers/misc/ocxl/config.c b/drivers/misc/ocxl/config.c index a62e3d7db2bf..69cca341d446 100644 --- a/drivers/misc/ocxl/config.c +++ b/drivers/misc/ocxl/config.c @@ -568,6 +568,10 @@ static int read_afu_lpc_memory_info(struct pci_dev *dev, afu->special_purpose_mem_size = total_mem_size - lpc_mem_size; } + + dev_info(&dev->dev, "Probed LPC memory of %#llx bytes and special purpose memory of %#llx bytes\n", +afu->lpc_mem_size, afu->special_purpose_mem_size); + return 0; } -- 2.24.1 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v4 01/25] powerpc/powernv: Add OPAL calls for LPC memory alloc/release
Add OPAL calls for LPC memory alloc/release Signed-off-by: Alastair D'Silva Acked-by: Andrew Donnellan Acked-by: Frederic Barrat --- arch/powerpc/include/asm/opal-api.h| 2 ++ arch/powerpc/include/asm/opal.h| 2 ++ arch/powerpc/platforms/powernv/opal-call.c | 2 ++ 3 files changed, 6 insertions(+) diff --git a/arch/powerpc/include/asm/opal-api.h b/arch/powerpc/include/asm/opal-api.h index c1f25a760eb1..9298e603001b 100644 --- a/arch/powerpc/include/asm/opal-api.h +++ b/arch/powerpc/include/asm/opal-api.h @@ -208,6 +208,8 @@ #define OPAL_HANDLE_HMI2 166 #defineOPAL_NX_COPROC_INIT 167 #define OPAL_XIVE_GET_VP_STATE 170 +#define OPAL_NPU_MEM_ALLOC 171 +#define OPAL_NPU_MEM_RELEASE 172 #define OPAL_MPIPL_UPDATE 173 #define OPAL_MPIPL_REGISTER_TAG174 #define OPAL_MPIPL_QUERY_TAG 175 diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h index 9986ac34b8e2..301fea46c7ca 100644 --- a/arch/powerpc/include/asm/opal.h +++ b/arch/powerpc/include/asm/opal.h @@ -39,6 +39,8 @@ int64_t opal_npu_spa_clear_cache(uint64_t phb_id, uint32_t bdfn, uint64_t PE_handle); int64_t opal_npu_tl_set(uint64_t phb_id, uint32_t bdfn, long cap, uint64_t rate_phys, uint32_t size); +int64_t opal_npu_mem_alloc(u64 phb_id, u32 bdfn, u64 size, __be64 *bar); +int64_t opal_npu_mem_release(u64 phb_id, u32 bdfn); int64_t opal_console_write(int64_t term_number, __be64 *length, const uint8_t *buffer); diff --git a/arch/powerpc/platforms/powernv/opal-call.c b/arch/powerpc/platforms/powernv/opal-call.c index 5cd0f52d258f..f26e58b72c04 100644 --- a/arch/powerpc/platforms/powernv/opal-call.c +++ b/arch/powerpc/platforms/powernv/opal-call.c @@ -287,6 +287,8 @@ OPAL_CALL(opal_pci_set_pbcq_tunnel_bar, OPAL_PCI_SET_PBCQ_TUNNEL_BAR); OPAL_CALL(opal_sensor_read_u64,OPAL_SENSOR_READ_U64); OPAL_CALL(opal_sensor_group_enable,OPAL_SENSOR_GROUP_ENABLE); OPAL_CALL(opal_nx_coproc_init, OPAL_NX_COPROC_INIT); +OPAL_CALL(opal_npu_mem_alloc, OPAL_NPU_MEM_ALLOC); +OPAL_CALL(opal_npu_mem_release,OPAL_NPU_MEM_RELEASE); OPAL_CALL(opal_mpipl_update, OPAL_MPIPL_UPDATE); OPAL_CALL(opal_mpipl_register_tag, OPAL_MPIPL_REGISTER_TAG); OPAL_CALL(opal_mpipl_query_tag,OPAL_MPIPL_QUERY_TAG); -- 2.24.1 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v4 05/25] ocxl: Address kernel doc errors & warnings
This patch addresses warnings and errors from the kernel doc scripts for
the OpenCAPI driver.
It also makes minor tweaks to make the docs more consistent.
Signed-off-by: Alastair D'Silva
Acked-by: Andrew Donnellan
---
drivers/misc/ocxl/config.c| 24
drivers/misc/ocxl/ocxl_internal.h | 9 +--
include/misc/ocxl.h | 96 ---
3 files changed, 55 insertions(+), 74 deletions(-)
diff --git a/drivers/misc/ocxl/config.c b/drivers/misc/ocxl/config.c
index c8e19bfb5ef9..a62e3d7db2bf 100644
--- a/drivers/misc/ocxl/config.c
+++ b/drivers/misc/ocxl/config.c
@@ -273,16 +273,16 @@ static int read_afu_info(struct pci_dev *dev, struct
ocxl_fn_config *fn,
}
/**
- * Read the template version from the AFU
- * dev: the device for the AFU
- * fn: the AFU offsets
- * len: outputs the template length
- * version: outputs the major<<8,minor version
+ * read_template_version() - Read the template version from the AFU
+ * @dev: the device for the AFU
+ * @fn: the AFU offsets
+ * @len: outputs the template length
+ * @version: outputs the major<<8,minor version
*
* Returns 0 on success, negative on failure
*/
static int read_template_version(struct pci_dev *dev, struct ocxl_fn_config
*fn,
- u16 *len, u16 *version)
+u16 *len, u16 *version)
{
u32 val32;
u8 major, minor;
@@ -476,16 +476,16 @@ static int validate_afu(struct pci_dev *dev, struct
ocxl_afu_config *afu)
}
/**
- * Populate AFU metadata regarding LPC memory
- * dev: the device for the AFU
- * fn: the AFU offsets
- * afu: the AFU struct to populate the LPC metadata into
+ * read_afu_lpc_memory_info() - Populate AFU metadata regarding LPC memory
+ * @dev: the device for the AFU
+ * @fn: the AFU offsets
+ * @afu: the AFU struct to populate the LPC metadata into
*
* Returns 0 on success, negative on failure
*/
static int read_afu_lpc_memory_info(struct pci_dev *dev,
- struct ocxl_fn_config *fn,
- struct ocxl_afu_config *afu)
+ struct ocxl_fn_config *fn,
+ struct ocxl_afu_config *afu)
{
int rc;
u32 val32;
diff --git a/drivers/misc/ocxl/ocxl_internal.h
b/drivers/misc/ocxl/ocxl_internal.h
index 345bf843a38e..198e4e4bc51d 100644
--- a/drivers/misc/ocxl/ocxl_internal.h
+++ b/drivers/misc/ocxl/ocxl_internal.h
@@ -122,11 +122,12 @@ int ocxl_config_check_afu_index(struct pci_dev *dev,
struct ocxl_fn_config *fn, int afu_idx);
/**
- * Update values within a Process Element
+ * ocxl_link_update_pe() - Update values within a Process Element
+ * @link_handle: the link handle associated with the process element
+ * @pasid: the PASID for the AFU context
+ * @tid: the new thread id for the process element
*
- * link_handle: the link handle associated with the process element
- * pasid: the PASID for the AFU context
- * tid: the new thread id for the process element
+ * Returns 0 on success
*/
int ocxl_link_update_pe(void *link_handle, int pasid, __u16 tid);
diff --git a/include/misc/ocxl.h b/include/misc/ocxl.h
index 0a762e387418..357ef1aadbc0 100644
--- a/include/misc/ocxl.h
+++ b/include/misc/ocxl.h
@@ -62,8 +62,7 @@ struct ocxl_context;
// Device detection & initialisation
/**
- * Open an OpenCAPI function on an OpenCAPI device
- *
+ * ocxl_function_open() - Open an OpenCAPI function on an OpenCAPI device
* @dev: The PCI device that contains the function
*
* Returns an opaque pointer to the function, or an error pointer (check with
IS_ERR)
@@ -71,8 +70,7 @@ struct ocxl_context;
struct ocxl_fn *ocxl_function_open(struct pci_dev *dev);
/**
- * Get the list of AFUs associated with a PCI function device
- *
+ * ocxl_function_afu_list() - Get the list of AFUs associated with a PCI
function device
* Returns a list of struct ocxl_afu *
*
* @fn: The OpenCAPI function containing the AFUs
@@ -80,8 +78,7 @@ struct ocxl_fn *ocxl_function_open(struct pci_dev *dev);
struct list_head *ocxl_function_afu_list(struct ocxl_fn *fn);
/**
- * Fetch an AFU instance from an OpenCAPI function
- *
+ * ocxl_function_fetch_afu() - Fetch an AFU instance from an OpenCAPI function
* @fn: The OpenCAPI function to get the AFU from
* @afu_idx: The index of the AFU to get
*
@@ -92,23 +89,20 @@ struct list_head *ocxl_function_afu_list(struct ocxl_fn
*fn);
struct ocxl_afu *ocxl_function_fetch_afu(struct ocxl_fn *fn, u8 afu_idx);
/**
- * Take a reference to an AFU
- *
+ * ocxl_afu_get() - Take a reference to an AFU
* @afu: The AFU to increment the reference count on
*/
void ocxl_afu_get(struct ocxl_afu *afu);
/**
- * Release a reference to an AFU
- *
+ * ocxl_afu_put() - Release a reference to an AFU
* @afu: The AFU to decrement the reference count on
*/
void ocxl_afu_put(struct ocxl_afu *afu);
/**
- * Get the configurati
[PATCH v4 02/25] mm/memory_hotplug: Allow check_hotplug_memory_addressable to be called from drivers
When setting up OpenCAPI connected persistent memory, the range check may
not be performed until quite late (or perhaps not at all, if the user does
not establish a DAX device).
This patch makes the range check callable so we can perform the check while
probing the OpenCAPI Persistent Memory device.
Signed-off-by: Alastair D'Silva
Reviewed-by: Andrew Donnellan
---
include/linux/memory_hotplug.h | 5 +
mm/memory_hotplug.c| 4 ++--
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
index f4d59155f3d4..9a19ae0d7e31 100644
--- a/include/linux/memory_hotplug.h
+++ b/include/linux/memory_hotplug.h
@@ -337,6 +337,11 @@ static inline void __remove_memory(int nid, u64 start, u64
size) {}
extern void set_zone_contiguous(struct zone *zone);
extern void clear_zone_contiguous(struct zone *zone);
+#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
+int check_hotplug_memory_addressable(unsigned long pfn,
+unsigned long nr_pages);
+#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
+
extern void __ref free_area_init_core_hotplug(int nid);
extern int __add_memory(int nid, u64 start, u64 size);
extern int add_memory(int nid, u64 start, u64 size);
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 0a54ffac8c68..14945f033594 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -276,8 +276,8 @@ static int check_pfn_span(unsigned long pfn, unsigned long
nr_pages,
return 0;
}
-static int check_hotplug_memory_addressable(unsigned long pfn,
- unsigned long nr_pages)
+int check_hotplug_memory_addressable(unsigned long pfn,
+unsigned long nr_pages)
{
const u64 max_addr = PFN_PHYS(pfn + nr_pages) - 1;
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v4 09/25] ocxl: Save the device serial number in ocxl_fn
This patch retrieves the serial number of the card and makes it available
to consumers of the ocxl driver via the ocxl_fn struct.
Signed-off-by: Alastair D'Silva
Acked-by: Frederic Barrat
Acked-by: Andrew Donnellan
---
drivers/misc/ocxl/config.c | 46 ++
include/misc/ocxl.h| 1 +
2 files changed, 47 insertions(+)
diff --git a/drivers/misc/ocxl/config.c b/drivers/misc/ocxl/config.c
index 69cca341d446..2b835e57daca 100644
--- a/drivers/misc/ocxl/config.c
+++ b/drivers/misc/ocxl/config.c
@@ -71,6 +71,51 @@ static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8
afu_idx)
return 0;
}
+/**
+ * get_function_0() - Find a related PCI device (function 0)
+ * @device: PCI device to match
+ *
+ * Returns a pointer to the related device, or null if not found
+ */
+static struct pci_dev *get_function_0(struct pci_dev *dev)
+{
+ unsigned int devfn = PCI_DEVFN(PCI_SLOT(dev->devfn), 0);
+
+ return pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),
+ dev->bus->number, devfn);
+}
+
+static void read_serial(struct pci_dev *dev, struct ocxl_fn_config *fn)
+{
+ u32 low, high;
+ int pos;
+
+ pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN);
+ if (pos) {
+ pci_read_config_dword(dev, pos + 0x04, &low);
+ pci_read_config_dword(dev, pos + 0x08, &high);
+
+ fn->serial = low | ((u64)high) << 32;
+
+ return;
+ }
+
+ if (PCI_FUNC(dev->devfn) != 0) {
+ struct pci_dev *related = get_function_0(dev);
+
+ if (!related) {
+ fn->serial = 0;
+ return;
+ }
+
+ read_serial(related, fn);
+ pci_dev_put(related);
+ return;
+ }
+
+ fn->serial = 0;
+}
+
static void read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn)
{
u16 val;
@@ -208,6 +253,7 @@ int ocxl_config_read_function(struct pci_dev *dev, struct
ocxl_fn_config *fn)
int rc;
read_pasid(dev, fn);
+ read_serial(dev, fn);
rc = read_dvsec_tl(dev, fn);
if (rc) {
diff --git a/include/misc/ocxl.h b/include/misc/ocxl.h
index d8b0b4d46bfb..b8514dc64bd0 100644
--- a/include/misc/ocxl.h
+++ b/include/misc/ocxl.h
@@ -46,6 +46,7 @@ struct ocxl_fn_config {
int dvsec_afu_info_pos; /* offset of the AFU information DVSEC */
s8 max_pasid_log;
s8 max_afu_index;
+ u64 serial;
};
enum ocxl_endian {
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v4 14/25] nvdimm/ocxl: Add support for Admin commands
Admin commands for these devices are the primary means of interacting
with the device controller to provide functionality beyond the load/store
capabilities offered via the NPU.
For example, SMART data, firmware update, and device error logs are
implemented via admin commands.
This patch requests the metadata required to issue admin commands, as well
as some helper functions to construct and check the completion of the
commands.
Signed-off-by: Alastair D'Silva
---
drivers/nvdimm/ocxl/main.c | 65 ++
drivers/nvdimm/ocxl/ocxlpmem.h | 50 -
drivers/nvdimm/ocxl/ocxlpmem_internal.c | 261
3 files changed, 375 insertions(+), 1 deletion(-)
diff --git a/drivers/nvdimm/ocxl/main.c b/drivers/nvdimm/ocxl/main.c
index be76acd33d74..8db573036423 100644
--- a/drivers/nvdimm/ocxl/main.c
+++ b/drivers/nvdimm/ocxl/main.c
@@ -217,6 +217,58 @@ static int register_lpc_mem(struct ocxlpmem *ocxlpmem)
return 0;
}
+/**
+ * extract_command_metadata() - Extract command data from MMIO & save it for
further use
+ * @ocxlpmem: the device metadata
+ * @offset: The base address of the command data structures (address of CREQO)
+ * @command_metadata: A pointer to the command metadata to populate
+ * Return: 0 on success, negative on failure
+ */
+static int extract_command_metadata(struct ocxlpmem *ocxlpmem, u32 offset,
+ struct command_metadata *command_metadata)
+{
+ int rc;
+ u64 tmp;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, offset,
+OCXL_LITTLE_ENDIAN, &tmp);
+ if (rc)
+ return rc;
+
+ command_metadata->request_offset = tmp >> 32;
+ command_metadata->response_offset = tmp & 0x;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, offset + 8,
+OCXL_LITTLE_ENDIAN, &tmp);
+ if (rc)
+ return rc;
+
+ command_metadata->data_offset = tmp >> 32;
+ command_metadata->data_size = tmp & 0x;
+
+ command_metadata->id = 0;
+
+ return 0;
+}
+
+/**
+ * setup_command_metadata() - Set up the command metadata
+ * @ocxlpmem: the device metadata
+ */
+static int setup_command_metadata(struct ocxlpmem *ocxlpmem)
+{
+ int rc;
+
+ mutex_init(&ocxlpmem->admin_command.lock);
+
+ rc = extract_command_metadata(ocxlpmem, GLOBAL_MMIO_ACMA_CREQO,
+ &ocxlpmem->admin_command);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+
/**
* allocate_minor() - Allocate a minor number to use for an OpenCAPI pmem
device
* @ocxlpmem: the device metadata
@@ -421,6 +473,14 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
ocxlpmem->pdev = pci_dev_get(pdev);
+ ocxlpmem->timeouts[ADMIN_COMMAND_ERRLOG] = 2000; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_HEARTBEAT] = 100; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_SMART] = 100; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_CONTROLLER_DUMP] = 1000; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_CONTROLLER_STATS] = 100; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_SHUTDOWN] = 1000; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_FW_UPDATE] = 16000; // ms
+
pci_set_drvdata(pdev, ocxlpmem);
ocxlpmem->ocxl_fn = ocxl_function_open(pdev);
@@ -467,6 +527,11 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
goto err;
}
+ if (setup_command_metadata(ocxlpmem)) {
+ dev_err(&pdev->dev, "Could not read command metadata\n");
+ goto err;
+ }
+
elapsed = 0;
timeout = ocxlpmem->readiness_timeout +
ocxlpmem->memory_available_timeout;
diff --git a/drivers/nvdimm/ocxl/ocxlpmem.h b/drivers/nvdimm/ocxl/ocxlpmem.h
index 3eadbe19f6d0..b72b3f909fc3 100644
--- a/drivers/nvdimm/ocxl/ocxlpmem.h
+++ b/drivers/nvdimm/ocxl/ocxlpmem.h
@@ -7,6 +7,7 @@
#include
#define LABEL_AREA_SIZEBIT_ULL(PA_SECTION_SHIFT)
+#define DEFAULT_TIMEOUT 100
#define GLOBAL_MMIO_CHI0x000
#define GLOBAL_MMIO_CHIC 0x008
@@ -57,6 +58,7 @@
#define GLOBAL_MMIO_HCI_CONTROLLER_DUMP_COLLECTED BIT_ULL(5) // CDC
#define GLOBAL_MMIO_HCI_REQ_HEALTH_PERFBIT_ULL(6) //
CHPD
+// must be maintained with admin_command_names in ocxlpmem_internal.c
#define ADMIN_COMMAND_HEARTBEAT0x00u
#define ADMIN_COMMAND_SHUTDOWN 0x01u
#define ADMIN_COMMAND_FW_UPDATE0x02u
@@ -68,6 +70,13 @@
#define ADMIN_COMMAND_CMD_CAPS 0x08u
#define ADMIN_COMMAND_MAX 0x08u
+#define NS_COMMAND_SECURE_ERASE0x20ull
+
+#define NS_RESPONSE_SECURE_ERASE_ACCESSIBLE_SUCCESS 0x20
+#d
[PATCH v4 00/25] Add support for OpenCAPI Persistent Memory devices
Improve patch description and remove redundant default - "nvdimm: Add driver for OpenCAPI Storage Class Memory" - Mark a few funcs as static as identified by the 0day bot - Add OCXL dependancies to OCXL_SCM - Use memcpy_mcsafe in scm_ndctl_config_read - Rename scm_foo_offset_0x00 to scm_foo_header_parse & add docs - Name DIMM attribs "ocxl" rather than "scm" - Split out into base + many feature patches - "powerpc: Enable OpenCAPI Storage Class Memory driver on bare metal" - Build DEV_DAX & friends as modules - "ocxl: Conditionally bind SCM devices to the generic OCXL driver" - Patch dropped (easy enough to maintain this out of tree for development) - "ocxl: Tally up the LPC memory on a link & allow it to be mapped" - Add a warning if an unmatched lpc_release is called - "ocxl: Add functions to map/unmap LPC memory" - Use EXPORT_SYMBOL_GPL Alastair D'Silva (25): powerpc/powernv: Add OPAL calls for LPC memory alloc/release mm/memory_hotplug: Allow check_hotplug_memory_addressable to be called from drivers powerpc/powernv: Map & release OpenCAPI LPC memory ocxl: Remove unnecessary externs ocxl: Address kernel doc errors & warnings ocxl: Tally up the LPC memory on a link & allow it to be mapped ocxl: Add functions to map/unmap LPC memory ocxl: Emit a log message showing how much LPC memory was detected ocxl: Save the device serial number in ocxl_fn nvdimm: Add driver for OpenCAPI Persistent Memory powerpc: Enable the OpenCAPI Persistent Memory driver for powernv_defconfig nvdimm/ocxl: Add register addresses & status values to the header nvdimm/ocxl: Read the capability registers & wait for device ready nvdimm/ocxl: Add support for Admin commands nvdimm/ocxl: Register a character device for userspace to interact with nvdimm/ocxl: Implement the Read Error Log command nvdimm/ocxl: Add controller dump IOCTLs nvdimm/ocxl: Add an IOCTL to report controller statistics nvdimm/ocxl: Forward events to userspace nvdimm/ocxl: Add an IOCTL to request controller health & perf data nvdimm/ocxl: Implement the heartbeat command nvdimm/ocxl: Add debug IOCTLs nvdimm/ocxl: Expose SMART data via ndctl nvdimm/ocxl: Expose the serial number & firmware version in sysfs MAINTAINERS: Add myself & nvdimm/ocxl to ocxl .../userspace-api/ioctl/ioctl-number.rst |1 + MAINTAINERS |3 + arch/powerpc/configs/powernv_defconfig|5 + arch/powerpc/include/asm/opal-api.h |2 + arch/powerpc/include/asm/opal.h |2 + arch/powerpc/include/asm/pnv-ocxl.h | 42 +- arch/powerpc/platforms/powernv/ocxl.c | 43 + arch/powerpc/platforms/powernv/opal-call.c|2 + drivers/misc/ocxl/config.c| 74 +- drivers/misc/ocxl/core.c | 61 + drivers/misc/ocxl/link.c | 60 + drivers/misc/ocxl/ocxl_internal.h | 45 +- drivers/nvdimm/Kconfig|2 + drivers/nvdimm/Makefile |1 + drivers/nvdimm/ocxl/Kconfig | 21 + drivers/nvdimm/ocxl/Makefile |7 + drivers/nvdimm/ocxl/main.c| 1975 + drivers/nvdimm/ocxl/ocxlpmem.h| 197 ++ drivers/nvdimm/ocxl/ocxlpmem_internal.c | 280 +++ include/linux/memory_hotplug.h|5 + include/misc/ocxl.h | 122 +- include/uapi/linux/ndctl.h|1 + include/uapi/nvdimm/ocxlpmem.h| 127 ++ mm/memory_hotplug.c |4 +- 24 files changed, 2983 insertions(+), 99 deletions(-) create mode 100644 drivers/nvdimm/ocxl/Kconfig create mode 100644 drivers/nvdimm/ocxl/Makefile create mode 100644 drivers/nvdimm/ocxl/main.c create mode 100644 drivers/nvdimm/ocxl/ocxlpmem.h create mode 100644 drivers/nvdimm/ocxl/ocxlpmem_internal.c create mode 100644 include/uapi/nvdimm/ocxlpmem.h -- 2.24.1 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v4 11/25] powerpc: Enable the OpenCAPI Persistent Memory driver for powernv_defconfig
This patch enables the OpenCAPI Persistent Memory driver, as well as DAX support, for the 'powernv' defconfig. DAX is not a strict requirement for the functioning of the driver, but it is likely that a user will want to create a DAX device on top of their persistent memory device. Signed-off-by: Alastair D'Silva Reviewed-by: Andrew Donnellan --- arch/powerpc/configs/powernv_defconfig | 5 + 1 file changed, 5 insertions(+) diff --git a/arch/powerpc/configs/powernv_defconfig b/arch/powerpc/configs/powernv_defconfig index 71749377d164..921d77bbd3d2 100644 --- a/arch/powerpc/configs/powernv_defconfig +++ b/arch/powerpc/configs/powernv_defconfig @@ -348,3 +348,8 @@ CONFIG_KVM_BOOK3S_64=m CONFIG_KVM_BOOK3S_64_HV=m CONFIG_VHOST_NET=m CONFIG_PRINTK_TIME=y +CONFIG_ZONE_DEVICE=y +CONFIG_OCXL_PMEM=m +CONFIG_DEV_DAX=m +CONFIG_DEV_DAX_PMEM=m +CONFIG_FS_DAX=y -- 2.24.1 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v4 07/25] ocxl: Add functions to map/unmap LPC memory
Add functions to map/unmap LPC memory
Signed-off-by: Alastair D'Silva
Acked-by: Frederic Barrat
---
drivers/misc/ocxl/core.c | 51 +++
drivers/misc/ocxl/ocxl_internal.h | 3 ++
include/misc/ocxl.h | 21 +
3 files changed, 75 insertions(+)
diff --git a/drivers/misc/ocxl/core.c b/drivers/misc/ocxl/core.c
index 2531c6cf19a0..75ff14e3882a 100644
--- a/drivers/misc/ocxl/core.c
+++ b/drivers/misc/ocxl/core.c
@@ -210,6 +210,56 @@ static void unmap_mmio_areas(struct ocxl_afu *afu)
release_fn_bar(afu->fn, afu->config.global_mmio_bar);
}
+int ocxl_afu_map_lpc_mem(struct ocxl_afu *afu)
+{
+ struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
+
+ if ((afu->config.lpc_mem_size + afu->config.special_purpose_mem_size)
== 0)
+ return 0;
+
+ afu->lpc_base_addr = ocxl_link_lpc_map(afu->fn->link, dev);
+ if (afu->lpc_base_addr == 0)
+ return -EINVAL;
+
+ if (afu->config.lpc_mem_size > 0) {
+ afu->lpc_res.start = afu->lpc_base_addr +
afu->config.lpc_mem_offset;
+ afu->lpc_res.end = afu->lpc_res.start +
afu->config.lpc_mem_size - 1;
+ }
+
+ if (afu->config.special_purpose_mem_size > 0) {
+ afu->special_purpose_res.start = afu->lpc_base_addr +
+
afu->config.special_purpose_mem_offset;
+ afu->special_purpose_res.end = afu->special_purpose_res.start +
+
afu->config.special_purpose_mem_size - 1;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(ocxl_afu_map_lpc_mem);
+
+struct resource *ocxl_afu_lpc_mem(struct ocxl_afu *afu)
+{
+ return &afu->lpc_res;
+}
+EXPORT_SYMBOL_GPL(ocxl_afu_lpc_mem);
+
+static void unmap_lpc_mem(struct ocxl_afu *afu)
+{
+ struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
+
+ if (afu->lpc_res.start || afu->special_purpose_res.start) {
+ void *link = afu->fn->link;
+
+ // only release the link when the the last consumer calls
release
+ ocxl_link_lpc_release(link, dev);
+
+ afu->lpc_res.start = 0;
+ afu->lpc_res.end = 0;
+ afu->special_purpose_res.start = 0;
+ afu->special_purpose_res.end = 0;
+ }
+}
+
static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev)
{
int rc;
@@ -251,6 +301,7 @@ static int configure_afu(struct ocxl_afu *afu, u8 afu_idx,
struct pci_dev *dev)
static void deconfigure_afu(struct ocxl_afu *afu)
{
+ unmap_lpc_mem(afu);
unmap_mmio_areas(afu);
reclaim_afu_pasid(afu);
reclaim_afu_actag(afu);
diff --git a/drivers/misc/ocxl/ocxl_internal.h
b/drivers/misc/ocxl/ocxl_internal.h
index 2d7575225bd7..7b975a89db7b 100644
--- a/drivers/misc/ocxl/ocxl_internal.h
+++ b/drivers/misc/ocxl/ocxl_internal.h
@@ -52,6 +52,9 @@ struct ocxl_afu {
void __iomem *global_mmio_ptr;
u64 pp_mmio_start;
void *private;
+ u64 lpc_base_addr; /* Covers both LPC & special purpose memory */
+ struct resource lpc_res;
+ struct resource special_purpose_res;
};
enum ocxl_context_status {
diff --git a/include/misc/ocxl.h b/include/misc/ocxl.h
index 357ef1aadbc0..d8b0b4d46bfb 100644
--- a/include/misc/ocxl.h
+++ b/include/misc/ocxl.h
@@ -203,6 +203,27 @@ int ocxl_irq_set_handler(struct ocxl_context *ctx, int
irq_id,
// AFU Metadata
+/**
+ * ocxl_afu_map_lpc_mem() - Map the LPC system & special purpose memory for an
AFU
+ * Do not call this during device discovery, as there may me multiple
+ * devices on a link, and the memory is mapped for the whole link, not
+ * just one device. It should only be called after all devices have
+ * registered their memory on the link.
+ *
+ * @afu: The AFU that has the LPC memory to map
+ *
+ * Returns 0 on success, negative on failure
+ */
+int ocxl_afu_map_lpc_mem(struct ocxl_afu *afu);
+
+/**
+ * ocxl_afu_lpc_mem() - Get the physical address range of LPC memory for an AFU
+ * @afu: The AFU associated with the LPC memory
+ *
+ * Returns a pointer to the resource struct for the physical address range
+ */
+struct resource *ocxl_afu_lpc_mem(struct ocxl_afu *afu);
+
/**
* ocxl_afu_config() - Get a pointer to the config for an AFU
* @afu: a pointer to the AFU to get the config for
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 24/27] powerpc/powernv/pmem: Expose SMART data via ndctl
On Thu, 2020-03-05 at 14:36 +1100, Andrew Donnellan wrote:
> On 21/2/20 2:27 pm, Alastair D'Silva wrote:
> > +static int ndctl_smart(struct ocxlpmem *ocxlpmem, struct
> > nd_cmd_pkg *pkg)
> > +{
> > + u32 length, i;
> > + struct nd_ocxl_smart *out;
> > + int rc;
> > +
> > + mutex_lock(&ocxlpmem->admin_command.lock);
> > +
> > + rc = admin_command_request(ocxlpmem, ADMIN_COMMAND_SMART);
> > + if (rc)
> > + goto out;
> > +
> > + rc = admin_command_execute(ocxlpmem);
> > + if (rc)
> > + goto out;
> > +
> > + rc = admin_command_complete_timeout(ocxlpmem,
> > ADMIN_COMMAND_SMART);
> > + if (rc < 0) {
> > + dev_err(&ocxlpmem->dev, "SMART timeout\n");
> > + goto out;
> > + }
> > +
> > + rc = admin_response(ocxlpmem);
> > + if (rc < 0)
> > + goto out;
> > + if (rc != STATUS_SUCCESS) {
> > + warn_status(ocxlpmem, "Unexpected status from SMART",
> > rc);
> > + goto out;
> > + }
> > +
> > + rc = smart_header_parse(ocxlpmem, &length);
> > + if (rc)
> > + goto out;
> > +
> > + pkg->nd_fw_size = length;
> > +
> > + length = min(length, pkg->nd_size_out); // bytes
> > + out = (struct nd_ocxl_smart *)pkg->nd_payload;
> > + // Each SMART attribute is 2 * 64 bits
> > + out->count = length / (2 * sizeof(u64)); // attributes
>
> From what I can tell - 8 bytes of nd_ocxl_smart are taken up for
> the
> count + reserved bytes, so this is going to potentially overrun the
> user
> buffer.
Ok
>
> > +
> > + for (i = 0; i < length; i += sizeof(u64)) {
>
> It might be neater to make i count up by 1 and then multiply by
> sizeof(u64) later.
>
I tried, it doesn't make much difference to the complexity, and makes
it less clear that we are stepping in 64bit chunks.
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > +ocxlpmem-
> > >admin_command.data_offset + sizeof(u64) + i,
>
> + 0x08 rather than + sizeof(u64) for consistency.
>
We use sizeof(u64) in the rest of this function.
> > +OCXL_LITTLE_ENDIAN,
> > +&out-
> > >attribs[i/sizeof(u64)]);
> > + if (rc)
> > + goto out;
> > + }
> > +
> > + rc = admin_response_handled(ocxlpmem);
> > + if (rc)
> > + goto out;
> > +
> > + rc = 0;
> > + goto out;
> > +
> > +out:
> > + mutex_unlock(&ocxlpmem->admin_command.lock);
> > + return rc;
> > +}
> > +
> > +static int ndctl_call(struct ocxlpmem *ocxlpmem, void *buf,
> > unsigned int buf_len)
> > +{
> > + struct nd_cmd_pkg *pkg = buf;
> > +
> > + if (buf_len < sizeof(struct nd_cmd_pkg)) {
> > + dev_err(&ocxlpmem->dev, "Invalid ND_CALL size=%u\n",
> > buf_len);
> > + return -EINVAL;
> > + }
> > +
> > + if (pkg->nd_family != NVDIMM_FAMILY_OCXL) {
> > + dev_err(&ocxlpmem->dev, "Invalid ND_CALL
> > family=0x%llx\n", pkg->nd_family);
> > + return -EINVAL;
> > + }
> > +
> > + switch (pkg->nd_command) {
> > + case ND_CMD_OCXL_SMART:
> > + ndctl_smart(ocxlpmem, pkg);
>
> Did you intend to dispose of the return code here?
Whoops.
>
> > + break;
> > +
> > + default:
> > + dev_err(&ocxlpmem->dev, "Invalid ND_CALL
> > command=0x%llx\n", pkg->nd_command);
> > + return -EINVAL;
> > + }
> > +
> > +
> > + return 0;
> > +}
> > +
> > static int ndctl(struct nvdimm_bus_descriptor *nd_desc,
> > struct nvdimm *nvdimm,
> > unsigned int cmd, void *buf, unsigned int buf_len, int
> > *cmd_rc)
> > @@ -88,6 +211,10 @@ static int ndctl(struct nvdimm_bus_descriptor
> > *nd_desc,
> > struct ocxlpmem *ocxlpmem = container_of(nd_desc, struct
> > ocxlpmem, bus_desc);
> >
> > switch (cmd) {
> > + case ND_CMD_CALL:
> > + *cmd_rc = ndctl_call(ocxlpmem, buf, buf_len);
> > + return 0;
> > +
> > case ND_CMD_GET_CONFIG_SIZE:
> > *cmd_rc = ndctl_config_size(buf);
> > return 0;
> > @@
Re: [PATCH v3 23/27] powerpc/powernv/pmem: Add debug IOCTLs
On Thu, 2020-03-05 at 14:11 +1100, Andrew Donnellan wrote:
> On 21/2/20 2:27 pm, Alastair D'Silva wrote:
> > From: Alastair D'Silva
> >
> > These IOCTLs provide low level access to the card to aid in
> > debugging
> > controller/FPGA firmware.
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > arch/powerpc/platforms/powernv/pmem/Kconfig | 6 +
> > arch/powerpc/platforms/powernv/pmem/ocxl.c | 249
> >
> > include/uapi/nvdimm/ocxl-pmem.h | 32 +++
> > 3 files changed, 287 insertions(+)
> >
> > diff --git a/arch/powerpc/platforms/powernv/pmem/Kconfig
> > b/arch/powerpc/platforms/powernv/pmem/Kconfig
> > index c5d927520920..3f44429d70c9 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/Kconfig
> > +++ b/arch/powerpc/platforms/powernv/pmem/Kconfig
> > @@ -12,4 +12,10 @@ config OCXL_PMEM
> >
> > Select N if unsure.
> >
> > +config OCXL_PMEM_DEBUG
> > + bool "OpenCAPI Persistent Memory debugging"
> > + depends on OCXL_PMEM
> > + help
> > + Enables low level IOCTLs for OpenCAPI Persistent Memory
> > firmware development
> > +
>
> How dangerous are these ioctls and does that need to be pointed out
> in
> this description?
Good point, I'll elaborate.
>
> > endif
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > index e01f6f9fc180..d4ce5e9e0521 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > @@ -1050,6 +1050,235 @@ int req_controller_health_perf(struct
> > ocxlpmem *ocxlpmem)
> > GLOBAL_MMIO_HCI_REQ_HEALTH_PERF);
> > }
> >
> > +#ifdef CONFIG_OCXL_PMEM_DEBUG
> > +/**
> > + * enable_fwdebug() - Enable FW debug on the controller
> > + * @ocxlpmem: the device metadata
> > + * Return: 0 on success, negative on failure
> > + */
> > +static int enable_fwdebug(const struct ocxlpmem *ocxlpmem)
> > +{
> > + return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu,
> > GLOBAL_MMIO_HCI,
> > + OCXL_LITTLE_ENDIAN,
> > + GLOBAL_MMIO_HCI_FW_DEBUG);
> > +}
> > +
> > +/**
> > + * disable_fwdebug() - Disable FW debug on the controller
> > + * @ocxlpmem: the device metadata
> > + * Return: 0 on success, negative on failure
> > + */
> > +static int disable_fwdebug(const struct ocxlpmem *ocxlpmem)
> > +{
> > + return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu,
> > GLOBAL_MMIO_HCIC,
> > + OCXL_LITTLE_ENDIAN,
> > + GLOBAL_MMIO_HCI_FW_DEBUG);
> > +}
> > +
> > +static int ioctl_fwdebug(struct ocxlpmem *ocxlpmem,
> > +struct ioctl_ocxl_pmem_fwdebug __user
> > *uarg)
> > +{
> > + struct ioctl_ocxl_pmem_fwdebug args;
> > + u64 val;
> > + int i;
> > + int rc;
> > +
> > + if (copy_from_user(&args, uarg, sizeof(args)))
> > + return -EFAULT;
> > +
> > + // Buffer size must be a multiple of 8
> > + if ((args.buf_size & 0x07))
> > + return -EINVAL;
> > +
> > + if (args.buf_size > ocxlpmem->admin_command.data_size)
> > + return -EINVAL;
> > +
> > + mutex_lock(&ocxlpmem->admin_command.lock);
> > +
> > + rc = enable_fwdebug(ocxlpmem);
> > + if (rc)
> > + goto out;
> > +
> > + rc = admin_command_request(ocxlpmem, ADMIN_COMMAND_FW_DEBUG);
> > + if (rc)
> > + goto out;
> > +
> > + // Write DebugAction & FunctionCode
> > + val = ((u64)args.debug_action << 56) | ((u64)args.function_code
> > << 40);
> > +
> > + rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
> > + ocxlpmem-
> > >admin_command.request_offset + 0x08,
> > + OCXL_LITTLE_ENDIAN, val);
> > + if (rc)
> > + goto out;
> > +
> > + rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
> > + ocxlpmem-
> > >admin_command.request_offset + 0x10,
> > + OCXL_LITTLE_ENDIAN,
> > args.debug_parameter_1);
> > + if (rc)
> > + goto out;
> > +
> > + rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
&
Re: [PATCH v3 19/27] powerpc/powernv/pmem: Add an IOCTL to report controller statistics
On Thu, 2020-03-05 at 11:46 +1100, Andrew Donnellan wrote:
> On 21/2/20 2:27 pm, Alastair D'Silva wrote:
> > From: Alastair D'Silva
> >
> > The controller can report a number of statistics that are useful
> > in evaluating the performance and reliability of the card.
> >
> > This patch exposes this information via an IOCTL.
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > arch/powerpc/platforms/powernv/pmem/ocxl.c | 185
> > +
> > include/uapi/nvdimm/ocxl-pmem.h| 17 ++
> > 2 files changed, 202 insertions(+)
> >
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > index 2cabafe1fc58..009d4fd29e7d 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > @@ -758,6 +758,186 @@ static int
> > ioctl_controller_dump_complete(struct ocxlpmem *ocxlpmem)
> > GLOBAL_MMIO_HCI_CONTROLLER_DUMP_COL
> > LECTED);
> > }
> >
> > +/**
> > + * controller_stats_header_parse() - Parse the first 64 bits of
> > the controller stats admin command response
> > + * @ocxlpmem: the device metadata
> > + * @length: out, returns the number of bytes in the response
> > (excluding the 64 bit header)
> > + */
> > +static int controller_stats_header_parse(struct ocxlpmem
> > *ocxlpmem,
> > + u32 *length)
> > +{
> > + int rc;
> > + u64 val;
> > +
> > + u16 data_identifier;
> > + u32 data_length;
> > +
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > +ocxlpmem-
> > >admin_command.data_offset,
> > +OCXL_LITTLE_ENDIAN, &val);
> > + if (rc)
> > + return rc;
> > +
> > + data_identifier = val >> 48;
> > + data_length = val & 0x;
> > +
> > + if (data_identifier != 0x4353) { // 'CS'
> > + dev_err(&ocxlpmem->dev,
> > + "Bad data identifier for controller stats,
> > expected 'CS', got '%-.*s'\n",
> > + 2, (char *)&data_identifier);
> > + return -EINVAL;
>
> Same comment as earlier patches re EINVAL
>
I don't think I've seen a comment yet on these particular blocks. Can
you suggest a better return value?
> > + }
> > +
> > + *length = data_length;
> > + return 0;
> > +}
> > +
> > +static int ioctl_controller_stats(struct ocxlpmem *ocxlpmem,
> > + struct
> > ioctl_ocxl_pmem_controller_stats __user *uarg)
> > +{
> > + struct ioctl_ocxl_pmem_controller_stats args;
> > + u32 length;
> > + int rc;
> > + u64 val;
> > +
> > + memset(&args, '\0', sizeof(args));
> > +
> > + mutex_lock(&ocxlpmem->admin_command.lock);
> > +
> > + rc = admin_command_request(ocxlpmem,
> > ADMIN_COMMAND_CONTROLLER_STATS);
> > + if (rc)
> > + goto out;
> > +
> > + rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
> > + ocxlpmem-
> > >admin_command.request_offset + 0x08,
> > + OCXL_LITTLE_ENDIAN, 0);
> > + if (rc)
> > + goto out;
> > +
> > + rc = admin_command_execute(ocxlpmem);
> > + if (rc)
> > + goto out;
> > +
> > +
> > + rc = admin_command_complete_timeout(ocxlpmem,
> > + ADMIN_COMMAND_CONTROLLER_ST
> > ATS);
> > + if (rc < 0) {
> > + dev_warn(&ocxlpmem->dev, "Controller stats timed
> > out\n");
> > + goto out;
> > + }
> > +
> > + rc = admin_response(ocxlpmem);
> > + if (rc < 0)
> > + goto out;
> > + if (rc != STATUS_SUCCESS) {
> > + warn_status(ocxlpmem,
> > + "Unexpected status from controller stats",
> > rc);
> > + goto out;
> > + }
> > +
> > + rc = controller_stats_header_parse(ocxlpmem, &length);
> > + if (rc)
> > + goto out;
> > +
> > + if (length != 0x140)
> > + warn_status(ocxlpmem,
> > + "Unexpected length for controller stats
> > data, expected 0x140, got 0x%x",
> > +
Re: [PATCH v3 23/27] powerpc/powernv/pmem: Add debug IOCTLs
On Wed, 2020-03-04 at 16:21 +0100, Frederic Barrat wrote:
>
> Le 21/02/2020 à 04:27, Alastair D'Silva a écrit :
> > From: Alastair D'Silva
> >
> > These IOCTLs provide low level access to the card to aid in
> > debugging
> > controller/FPGA firmware.
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > arch/powerpc/platforms/powernv/pmem/Kconfig | 6 +
> > arch/powerpc/platforms/powernv/pmem/ocxl.c | 249
> >
> > include/uapi/nvdimm/ocxl-pmem.h | 32 +++
> > 3 files changed, 287 insertions(+)
> >
> > diff --git a/arch/powerpc/platforms/powernv/pmem/Kconfig
> > b/arch/powerpc/platforms/powernv/pmem/Kconfig
> > index c5d927520920..3f44429d70c9 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/Kconfig
> > +++ b/arch/powerpc/platforms/powernv/pmem/Kconfig
> > @@ -12,4 +12,10 @@ config OCXL_PMEM
> >
> > Select N if unsure.
> >
> > +config OCXL_PMEM_DEBUG
> > + bool "OpenCAPI Persistent Memory debugging"
> > + depends on OCXL_PMEM
> > + help
> > + Enables low level IOCTLs for OpenCAPI Persistent Memory
> > firmware development
> > +
> > endif
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > index e01f6f9fc180..d4ce5e9e0521 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > @@ -1050,6 +1050,235 @@ int req_controller_health_perf(struct
> > ocxlpmem *ocxlpmem)
> > GLOBAL_MMIO_HCI_REQ_HEALTH_PERF);
> > }
> >
> > +#ifdef CONFIG_OCXL_PMEM_DEBUG
> > +/**
> > + * enable_fwdebug() - Enable FW debug on the controller
> > + * @ocxlpmem: the device metadata
> > + * Return: 0 on success, negative on failure
> > + */
> > +static int enable_fwdebug(const struct ocxlpmem *ocxlpmem)
> > +{
> > + return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu,
> > GLOBAL_MMIO_HCI,
> > + OCXL_LITTLE_ENDIAN,
> > + GLOBAL_MMIO_HCI_FW_DEBUG);
> > +}
> > +
> > +/**
> > + * disable_fwdebug() - Disable FW debug on the controller
> > + * @ocxlpmem: the device metadata
> > + * Return: 0 on success, negative on failure
> > + */
> > +static int disable_fwdebug(const struct ocxlpmem *ocxlpmem)
> > +{
> > + return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu,
> > GLOBAL_MMIO_HCIC,
> > + OCXL_LITTLE_ENDIAN,
> > + GLOBAL_MMIO_HCI_FW_DEBUG);
> > +}
> > +
> > +static int ioctl_fwdebug(struct ocxlpmem *ocxlpmem,
> > +struct ioctl_ocxl_pmem_fwdebug __user
> > *uarg)
> > +{
> > + struct ioctl_ocxl_pmem_fwdebug args;
> > + u64 val;
> > + int i;
> > + int rc;
> > +
> > + if (copy_from_user(&args, uarg, sizeof(args)))
> > + return -EFAULT;
> > +
> > + // Buffer size must be a multiple of 8
> > + if ((args.buf_size & 0x07))
> > + return -EINVAL;
> > +
> > + if (args.buf_size > ocxlpmem->admin_command.data_size)
> > + return -EINVAL;
> > +
> > + mutex_lock(&ocxlpmem->admin_command.lock);
> > +
> > + rc = enable_fwdebug(ocxlpmem);
> > + if (rc)
> > + goto out;
> > +
> > + rc = admin_command_request(ocxlpmem, ADMIN_COMMAND_FW_DEBUG);
> > + if (rc)
> > + goto out;
> > +
> > + // Write DebugAction & FunctionCode
> > + val = ((u64)args.debug_action << 56) | ((u64)args.function_code
> > << 40);
> > +
> > + rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
> > + ocxlpmem-
> > >admin_command.request_offset + 0x08,
> > + OCXL_LITTLE_ENDIAN, val);
> > + if (rc)
> > + goto out;
> > +
> > + rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
> > + ocxlpmem-
> > >admin_command.request_offset + 0x10,
> > + OCXL_LITTLE_ENDIAN,
> > args.debug_parameter_1);
> > + if (rc)
> > + goto out;
> > +
> > + rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
> > + ocxlpmem-
> > >admin_command.request_offset + 0x18,
> > +
Re: [PATCH v3 19/27] powerpc/powernv/pmem: Add an IOCTL to report controller statistics
On Wed, 2020-03-04 at 10:25 +0100, Frederic Barrat wrote:
>
> Le 21/02/2020 à 04:27, Alastair D'Silva a écrit :
> > From: Alastair D'Silva
> >
> > The controller can report a number of statistics that are useful
> > in evaluating the performance and reliability of the card.
> >
> > This patch exposes this information via an IOCTL.
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > arch/powerpc/platforms/powernv/pmem/ocxl.c | 185
> > +
> > include/uapi/nvdimm/ocxl-pmem.h| 17 ++
> > 2 files changed, 202 insertions(+)
> >
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > index 2cabafe1fc58..009d4fd29e7d 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > @@ -758,6 +758,186 @@ static int
> > ioctl_controller_dump_complete(struct ocxlpmem *ocxlpmem)
> > GLOBAL_MMIO_HCI_CONTROLLER_DUMP_COL
> > LECTED);
> > }
> >
> > +/**
> > + * controller_stats_header_parse() - Parse the first 64 bits of
> > the controller stats admin command response
> > + * @ocxlpmem: the device metadata
> > + * @length: out, returns the number of bytes in the response
> > (excluding the 64 bit header)
> > + */
> > +static int controller_stats_header_parse(struct ocxlpmem
> > *ocxlpmem,
> > + u32 *length)
> > +{
> > + int rc;
> > + u64 val;
> > +
>
> unexpected empty line
>
Ok
>
> > + u16 data_identifier;
> > + u32 data_length;
> > +
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > +ocxlpmem-
> > >admin_command.data_offset,
> > +OCXL_LITTLE_ENDIAN, &val);
> > + if (rc)
> > + return rc;
> > +
> > + data_identifier = val >> 48;
> > + data_length = val & 0x;
> > +
> > + if (data_identifier != 0x4353) { // 'CS'
> > + dev_err(&ocxlpmem->dev,
> > + "Bad data identifier for controller stats,
> > expected 'CS', got '%-.*s'\n",
> > + 2, (char *)&data_identifier);
>
>
> Wow, I'm clueless what that string format looks like :-)
> 2 arguments? Did you check the kernel string formatter does what you
> want?
> You may consider unifying the format though, the error log patch uses
> a
> simpler (better?) format for a similar message.
>
Sorry, force of habit from my old job where we dealt with a lot of
variable length, non-NULL terminated buffers. FYI - it takes the string
length from the first argument.
I'll change it to a fixed length string like the others :)
>
>
> > + return -EINVAL;
> > + }
> > +
> > + *length = data_length;
> > + return 0;
> > +}
> > +
> > +static int ioctl_controller_stats(struct ocxlpmem *ocxlpmem,
> > + struct
> > ioctl_ocxl_pmem_controller_stats __user *uarg)
> > +{
> > + struct ioctl_ocxl_pmem_controller_stats args;
> > + u32 length;
> > + int rc;
> > + u64 val;
> > +
> > + memset(&args, '\0', sizeof(args));
> > +
> > + mutex_lock(&ocxlpmem->admin_command.lock);
> > +
> > + rc = admin_command_request(ocxlpmem,
> > ADMIN_COMMAND_CONTROLLER_STATS);
> > + if (rc)
> > + goto out;
> > +
> > + rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
> > + ocxlpmem-
> > >admin_command.request_offset + 0x08,
> > + OCXL_LITTLE_ENDIAN, 0);
> > + if (rc)
> > + goto out;
> > +
> > + rc = admin_command_execute(ocxlpmem);
> > + if (rc)
> > + goto out;
> > +
> > +
> > + rc = admin_command_complete_timeout(ocxlpmem,
> > + ADMIN_COMMAND_CONTROLLER_ST
> > ATS);
> > + if (rc < 0) {
> > + dev_warn(&ocxlpmem->dev, "Controller stats timed
> > out\n");
> > + goto out;
> > + }
> > +
> > + rc = admin_response(ocxlpmem);
> > + if (rc < 0)
> > + goto out;
> > + if (rc != STATUS_SUCCESS) {
> > + warn_status(ocxlpmem,
> > + "Unexpected status from controller stats&qu
Re: [PATCH v3 21/27] powerpc/powernv/pmem: Add an IOCTL to request controller health & perf data
On Wed, 2020-03-04 at 12:06 +0100, Frederic Barrat wrote: > > Le 28/02/2020 à 07:12, Andrew Donnellan a écrit : > > On 21/2/20 2:27 pm, Alastair D'Silva wrote: > > > From: Alastair D'Silva > > > > > > When health & performance data is requested from the controller, > > > it responds with an error log containing the requested > > > information. > > > > > > This patch allows the request to me issued via an IOCTL. > > > > A better explanation would be good - this IOCTL triggers a request > > to > > the controller to collect controller health/perf data, and the > > controller will later respond with an error log that can be picked > > up > > via the error log IOCTL that you've defined earlier. > > And even more precisely (to also check my understanding): > > > this IOCTL triggers a request to > > the controller to collect controller health/perf data, and the > > controller will later respond > > by raising an interrupt to let the user app know that > > > an error log that can be picked up > > via the error log IOCTL that you've defined earlier. > > > The rest of the patch looks ok to me. > >Fred Ok -- Alastair D'Silva Open Source Developer Linux Technology Centre, IBM Australia mob: 0423 762 819 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 20/27] powerpc/powernv/pmem: Forward events to userspace
On Wed, 2020-03-04 at 12:00 +0100, Frederic Barrat wrote:
>
> Le 21/02/2020 à 04:27, Alastair D'Silva a écrit :
> > From: Alastair D'Silva
> >
> > Some of the interrupts that the card generates are better handled
> > by the userspace daemon, in particular:
> > Controller Hardware/Firmware Fatal
> > Controller Dump Available
> > Error Log available
> >
> > This patch allows a userspace application to register an eventfd
> > with
> > the driver via SCM_IOCTL_EVENTFD to receive notifications of these
> > interrupts.
> >
> > Userspace can then identify what events have occurred by calling
> > SCM_IOCTL_EVENT_CHECK and checking against the SCM_IOCTL_EVENT_FOO
> > masks.
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > arch/powerpc/platforms/powernv/pmem/ocxl.c| 216
> > ++
> > .../platforms/powernv/pmem/ocxl_internal.h| 5 +
> > include/uapi/nvdimm/ocxl-pmem.h | 16 ++
> > 3 files changed, 237 insertions(+)
> >
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > index 009d4fd29e7d..e46696d3cc36 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > @@ -10,6 +10,7 @@
> > #include
> > #include
> > #include
> > +#include
> > #include
> > #include
> > #include
> > @@ -335,11 +336,22 @@ static void free_ocxlpmem(struct ocxlpmem
> > *ocxlpmem)
> > {
> > int rc;
> >
> > + // Disable doorbells
> > + (void)ocxl_global_mmio_set64(ocxlpmem->ocxl_afu,
> > GLOBAL_MMIO_CHIEC,
> > +OCXL_LITTLE_ENDIAN,
> > +GLOBAL_MMIO_CHI_ALL);
> > +
> > if (ocxlpmem->nvdimm_bus)
> > nvdimm_bus_unregister(ocxlpmem->nvdimm_bus);
> >
> > free_minor(ocxlpmem);
> >
> > + if (ocxlpmem->irq_addr[1])
> > + iounmap(ocxlpmem->irq_addr[1]);
> > +
> > + if (ocxlpmem->irq_addr[0])
> > + iounmap(ocxlpmem->irq_addr[0]);
> > +
> > if (ocxlpmem->cdev.owner)
> > cdev_del(&ocxlpmem->cdev);
> >
> > @@ -443,6 +455,11 @@ static int file_release(struct inode *inode,
> > struct file *file)
> > {
> > struct ocxlpmem *ocxlpmem = file->private_data;
> >
> > + if (ocxlpmem->ev_ctx) {
> > + eventfd_ctx_put(ocxlpmem->ev_ctx);
> > + ocxlpmem->ev_ctx = NULL;
> > + }
> > +
> > ocxlpmem_put(ocxlpmem);
> > return 0;
> > }
> > @@ -938,6 +955,51 @@ static int ioctl_controller_stats(struct
> > ocxlpmem *ocxlpmem,
> > return rc;
> > }
> >
> > +static int ioctl_eventfd(struct ocxlpmem *ocxlpmem,
> > +struct ioctl_ocxl_pmem_eventfd __user *uarg)
> > +{
> > + struct ioctl_ocxl_pmem_eventfd args;
> > +
> > + if (copy_from_user(&args, uarg, sizeof(args)))
> > + return -EFAULT;
> > +
> > + if (ocxlpmem->ev_ctx)
> > + return -EINVAL;
>
> EBUSY?
>
Ok
>
> > +
> > + ocxlpmem->ev_ctx = eventfd_ctx_fdget(args.eventfd);
> > + if (!ocxlpmem->ev_ctx)
> > + return -EFAULT;
>
> Why not use what eventfd_ctx_fdget() returned? (through some
> IS_ERR()
> and PTR_ERR() convolution)
>
Ok
>
> > +
> > + return 0;
> > +}
> > +
> > +static int ioctl_event_check(struct ocxlpmem *ocxlpmem, u64 __user
> > *uarg)
> > +{
> > + u64 val = 0;
> > + int rc;
> > + u64 chi = 0;
> > +
> > + rc = ocxlpmem_chi(ocxlpmem, &chi);
> > + if (rc < 0)
> > + return rc;
> > +
> > + if (chi & GLOBAL_MMIO_CHI_ELA)
> > + val |= IOCTL_OCXL_PMEM_EVENT_ERROR_LOG_AVAILABLE;
> > +
> > + if (chi & GLOBAL_MMIO_CHI_CDA)
> > + val |= IOCTL_OCXL_PMEM_EVENT_CONTROLLER_DUMP_AVAILABLE;
> > +
> > + if (chi & GLOBAL_MMIO_CHI_CFFS)
> > + val |= IOCTL_OCXL_PMEM_EVENT_FIRMWARE_FATAL;
> > +
> > + if (chi & GLOBAL_MMIO_CHI_CHFS)
> > + val |= IOCTL_OCXL_PMEM_EVENT_HARDWARE_FATAL;
> > +
> > + rc = copy_to_user((u64 __user *) uarg, &val, sizeof(val));
> > +
>
> copy_to_user doesn't return an errno. Should be:
>
> if (copy_to_u
Re: [PATCH v3 18/27] powerpc/powernv/pmem: Add controller dump IOCTLs
On Wed, 2020-03-04 at 17:53 +1100, Andrew Donnellan wrote:
> On 21/2/20 2:27 pm, Alastair D'Silva wrote:
> > +static int ioctl_controller_dump_data(struct ocxlpmem *ocxlpmem,
> > + struct ioctl_ocxl_pmem_controller_dump_data __user
> > *uarg)
> > +{
> > + struct ioctl_ocxl_pmem_controller_dump_data args;
> > + u16 i;
> > + u64 val;
> > + int rc;
> > +
> > + if (copy_from_user(&args, uarg, sizeof(args)))
> > + return -EFAULT;
> > +
> > + if (args.buf_size % 8)
> > + return -EINVAL;
> > +
> > + if (args.buf_size > ocxlpmem->admin_command.data_size)
> > + return -EINVAL;
> > +
> > + mutex_lock(&ocxlpmem->admin_command.lock);
> > +
> > + rc = admin_command_request(ocxlpmem,
> > ADMIN_COMMAND_CONTROLLER_DUMP);
> > + if (rc)
> > + goto out;
> > +
> > + val = ((u64)args.offset) << 32;
> > + val |= args.buf_size;
> > + rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
> > + ocxlpmem-
> > >admin_command.request_offset + 0x08,
> > + OCXL_LITTLE_ENDIAN, val);
> > + if (rc)
> > + goto out;
> > +
> > + rc = admin_command_execute(ocxlpmem);
> > + if (rc)
> > + goto out;
> > +
> > + rc = admin_command_complete_timeout(ocxlpmem,
> > + ADMIN_COMMAND_CONTROLLER_DU
> > MP);
> > + if (rc < 0) {
> > + dev_warn(&ocxlpmem->dev, "Controller dump timed
> > out\n");
> > + goto out;
> > + }
> > +
> > + rc = admin_response(ocxlpmem);
> > + if (rc < 0)
> > + goto out;
> > + if (rc != STATUS_SUCCESS) {
> > + warn_status(ocxlpmem,
> > + "Unexpected status from retrieve error
> > log",
>
> Controller dump
>
Ok
> > + rc);
> > + goto out;
> > + }
> > +
> > + for (i = 0; i < args.buf_size; i += 8) {
> > + u64 val;
> > +
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > +ocxlpmem-
> > >admin_command.data_offset + i,
> > +OCXL_HOST_ENDIAN, &val);
>
> Is a controller dump something where we want to do endian swapping?
>
No, we just have raw binary data that we want to pass through.
OCXL_HOST_ENDIAN does no swapping.
> Any reason we're not doing the usual check of the data identifier,
> additional data length etc?
>
I'll add that
> > + if (rc)
> > + goto out;
> > +
> > + if (copy_to_user(&args.buf[i], &val, sizeof(u64))) {
> > + rc = -EFAULT;
> > + goto out;
> > + }
> > + }
> > +
> > + if (copy_to_user(uarg, &args, sizeof(args))) {
> > + rc = -EFAULT;
> > + goto out;
> > + }
> > +
> > + rc = admin_response_handled(ocxlpmem);
> > + if (rc)
> > + goto out;
> > +
> > +out:
> > + mutex_unlock(&ocxlpmem->admin_command.lock);
> > + return rc;
> > +}
> > +
> > +int request_controller_dump(struct ocxlpmem *ocxlpmem)
> > +{
> > + int rc;
> > + u64 busy = 1;
> > +
> > + rc = ocxl_global_mmio_set64(ocxlpmem->ocxl_afu,
> > GLOBAL_MMIO_CHIC,
> > + OCXL_LITTLE_ENDIAN,
> > + GLOBAL_MMIO_CHI_CDA);
>
> This return code is ignored
>
> > +
> > +
> > + rc = ocxl_global_mmio_set64(ocxlpmem->ocxl_afu,
> > GLOBAL_MMIO_HCI,
> > + OCXL_LITTLE_ENDIAN,
> > + GLOBAL_MMIO_HCI_CONTROLLER_DUMP);
> > + if (rc)
> > + return rc;
> > +
> > + while (busy) {
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > +GLOBAL_MMIO_HCI,
> > +OCXL_LITTLE_ENDIAN,
> > &busy);
> > + if (rc)
> > + return rc;
> > +
> > + busy &= GLOBAL_MMIO_HCI_CONTROLLER_DUMP;
> > + cond_resched();
> > + }
> > +
> > + return 0;
> > +}
>
>
--
Alastair D'Silva
Open Source Developer
Linux Technology Centre, IBM Australia
mob: 0423 762 819
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 18/27] powerpc/powernv/pmem: Add controller dump IOCTLs
On Tue, 2020-03-03 at 19:04 +0100, Frederic Barrat wrote:
>
> Le 21/02/2020 à 04:27, Alastair D'Silva a écrit :
> > From: Alastair D'Silva
> >
> > This patch adds IOCTLs to allow userspace to request & fetch dumps
> > of the internal controller state.
> >
> > This is useful during debugging or when a fatal error on the
> > controller
> > has occurred.
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > arch/powerpc/platforms/powernv/pmem/ocxl.c | 132
> > +
> > include/uapi/nvdimm/ocxl-pmem.h| 15 +++
> > 2 files changed, 147 insertions(+)
> >
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > index 2b64504f9129..2cabafe1fc58 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > @@ -640,6 +640,124 @@ static int ioctl_error_log(struct ocxlpmem
> > *ocxlpmem,
> > return 0;
> > }
> >
> > +static int ioctl_controller_dump_data(struct ocxlpmem *ocxlpmem,
> > + struct ioctl_ocxl_pmem_controller_dump_data __user
> > *uarg)
> > +{
> > + struct ioctl_ocxl_pmem_controller_dump_data args;
> > + u16 i;
> > + u64 val;
> > + int rc;
> > +
> > + if (copy_from_user(&args, uarg, sizeof(args)))
> > + return -EFAULT;
> > +
> > + if (args.buf_size % 8)
> > + return -EINVAL;
> > +
> > + if (args.buf_size > ocxlpmem->admin_command.data_size)
> > + return -EINVAL;
> > +
> > + mutex_lock(&ocxlpmem->admin_command.lock);
> > +
> > + rc = admin_command_request(ocxlpmem,
> > ADMIN_COMMAND_CONTROLLER_DUMP);
> > + if (rc)
> > + goto out;
> > +
> > + val = ((u64)args.offset) << 32;
> > + val |= args.buf_size;
> > + rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
> > + ocxlpmem-
> > >admin_command.request_offset + 0x08,
> > + OCXL_LITTLE_ENDIAN, val);
> > + if (rc)
> > + goto out;
> > +
> > + rc = admin_command_execute(ocxlpmem);
> > + if (rc)
> > + goto out;
> > +
> > + rc = admin_command_complete_timeout(ocxlpmem,
> > + ADMIN_COMMAND_CONTROLLER_DU
> > MP);
> > + if (rc < 0) {
> > + dev_warn(&ocxlpmem->dev, "Controller dump timed
> > out\n");
> > + goto out;
> > + }
> > +
> > + rc = admin_response(ocxlpmem);
> > + if (rc < 0)
> > + goto out;
> > + if (rc != STATUS_SUCCESS) {
> > + warn_status(ocxlpmem,
> > + "Unexpected status from retrieve error
> > log",
> > + rc);
> > + goto out;
> > + }
>
>
> It would help if there was a comment indicating how the 3 ioctls are
> used. My understanding is that the userland is:
> - requesting the controller to prepare a state dump
> - then one or more ioctls to fetch the data. The number of calls
> required to get the full state really depends on the size of the
> buffer
> passed by user
> - a last ioctl to tell the controller that we're done, presumably to
> let
> it free some resources.
>
Ok, will add it to the blurb.
>
> > +
> > + for (i = 0; i < args.buf_size; i += 8) {
> > + u64 val;
> > +
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > +ocxlpmem-
> > >admin_command.data_offset + i,
> > +OCXL_HOST_ENDIAN, &val);
> > + if (rc)
> > + goto out;
> > +
> > + if (copy_to_user(&args.buf[i], &val, sizeof(u64))) {
> > + rc = -EFAULT;
> > + goto out;
> > + }
> > + }
> > +
> > + if (copy_to_user(uarg, &args, sizeof(args))) {
> > + rc = -EFAULT;
> > + goto out;
> > + }
> > +
> > + rc = admin_response_handled(ocxlpmem);
> > + if (rc)
> > + goto out;
> > +
> > +out:
> > + mutex_unlock(&ocxlpmem->admin_command.lock);
> > + return rc;
> > +}
> > +
> > +int request_controller_dump(struct ocxlpmem *ocxlpmem)
> &g
Re: [PATCH v3 17/27] powerpc/powernv/pmem: Implement the Read Error Log command
On Tue, 2020-03-03 at 11:36 +0100, Frederic Barrat wrote:
>
> Le 21/02/2020 à 04:27, Alastair D'Silva a écrit :
> > From: Alastair D'Silva
> >
> > The read error log command extracts information from the
> > controller's
> > internal error log.
> >
> > This patch exposes this information in 2 ways:
> > - During probe, if an error occurs & a log is available, print it
> > to the
> >console
> > - After probe, make the error log available to userspace via an
> > IOCTL.
> >Userspace is notified of pending error logs in a later patch
> >("powerpc/powernv/pmem: Forward events to userspace")
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > arch/powerpc/platforms/powernv/pmem/ocxl.c| 269
> > ++
> > .../platforms/powernv/pmem/ocxl_internal.h| 1 +
> > include/uapi/nvdimm/ocxl-pmem.h | 46 +++
> > 3 files changed, 316 insertions(+)
> > create mode 100644 include/uapi/nvdimm/ocxl-pmem.h
> >
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > index 63109a870d2c..2b64504f9129 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > @@ -447,10 +447,219 @@ static int file_release(struct inode *inode,
> > struct file *file)
> > return 0;
> > }
> >
> > +/**
> > + * error_log_header_parse() - Parse the first 64 bits of the error
> > log command response
> > + * @ocxlpmem: the device metadata
> > + * @length: out, returns the number of bytes in the response
> > (excluding the 64 bit header)
> > + */
> > +static int error_log_header_parse(struct ocxlpmem *ocxlpmem, u16
> > *length)
> > +{
> > + int rc;
> > + u64 val;
> > +
>
> Empty line in the middle of declarations
>
Ok
>
> > + u16 data_identifier;
> > + u32 data_length;
> > +
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > +ocxlpmem-
> > >admin_command.data_offset,
> > +OCXL_LITTLE_ENDIAN, &val);
> > + if (rc)
> > + return rc;
> > +
> > + data_identifier = val >> 48;
> > + data_length = val & 0x;
> > +
> > + if (data_identifier != 0x454C) { // 'EL'
> > + dev_err(&ocxlpmem->dev,
> > + "Bad data identifier for error log data,
> > expected 'EL', got '%2s' (%#x), data_length=%u\n",
> > + (char *)&data_identifier,
> > + (unsigned int)data_identifier, data_length);
> > + return -EINVAL;
> > + }
> > +
> > + *length = data_length;
> > + return 0;
> > +}
> > +
> > +static int error_log_offset_0x08(struct ocxlpmem *ocxlpmem,
> > +u32 *log_identifier, u32
> > *program_ref_code)
> > +{
> > + int rc;
> > + u64 val;
> > +
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > +ocxlpmem-
> > >admin_command.data_offset + 0x08,
> > +OCXL_LITTLE_ENDIAN, &val);
> > + if (rc)
> > + return rc;
> > +
> > + *log_identifier = val >> 32;
> > + *program_ref_code = val & 0x;
> > +
> > + return 0;
> > +}
> > +
> > +static int read_error_log(struct ocxlpmem *ocxlpmem,
> > + struct ioctl_ocxl_pmem_error_log *log, bool
> > buf_is_user)
> > +{
> > + u64 val;
> > + u16 user_buf_length;
> > + u16 buf_length;
> > + u16 i;
> > + int rc;
> > +
> > + if (log->buf_size % 8)
> > + return -EINVAL;
> > +
> > + rc = ocxlpmem_chi(ocxlpmem, &val);
> > + if (rc)
> > + goto out;
>
>
> "out" will unlock a mutex not yet taken.
>
Thanks, that should have been a return.
>
>
> > +
> > + if (!(val & GLOBAL_MMIO_CHI_ELA))
> > + return -EAGAIN;
> > +
> > + user_buf_length = log->buf_size;
> > +
> > + mutex_lock(&ocxlpmem->admin_command.lock);
> > +
> > + rc = admin_command_request(ocxlpmem, ADMIN_COMMAND_ERRLOG);
> > + if (rc)
> > + goto out;
> > +
> > + rc = admin_command_execute(ocxlpmem);
> > + if
Re: [PATCH v3 16/27] powerpc/powernv/pmem: Register a character device for userspace to interact with
On Tue, 2020-03-03 at 10:28 +0100, Frederic Barrat wrote:
>
> Le 21/02/2020 à 04:27, Alastair D'Silva a écrit :
> > From: Alastair D'Silva
> >
> > This patch introduces a character device (/dev/ocxl-scmX) which
> > further
> > patches will use to interact with userspace.
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > arch/powerpc/platforms/powernv/pmem/ocxl.c| 116
> > +-
> > .../platforms/powernv/pmem/ocxl_internal.h| 2 +
> > 2 files changed, 116 insertions(+), 2 deletions(-)
> >
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > index b8bd7e703b19..63109a870d2c 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > @@ -10,6 +10,7 @@
> > #include
> > #include
> > #include
> > +#include
> > #include
> > #include
> > #include "ocxl_internal.h"
> > @@ -339,6 +340,9 @@ static void free_ocxlpmem(struct ocxlpmem
> > *ocxlpmem)
> >
> > free_minor(ocxlpmem);
> >
> > + if (ocxlpmem->cdev.owner)
> > + cdev_del(&ocxlpmem->cdev);
> > +
> > if (ocxlpmem->metadata_addr)
> > devm_memunmap(&ocxlpmem->dev, ocxlpmem->metadata_addr);
> >
> > @@ -396,6 +400,70 @@ static int ocxlpmem_register(struct ocxlpmem
> > *ocxlpmem)
> > return device_register(&ocxlpmem->dev);
> > }
> >
> > +static void ocxlpmem_put(struct ocxlpmem *ocxlpmem)
> > +{
> > + put_device(&ocxlpmem->dev);
> > +}
> > +
> > +static struct ocxlpmem *ocxlpmem_get(struct ocxlpmem *ocxlpmem)
> > +{
> > + return (get_device(&ocxlpmem->dev) == NULL) ? NULL : ocxlpmem;
> > +}
> > +
> > +static struct ocxlpmem *find_and_get_ocxlpmem(dev_t devno)
> > +{
> > + struct ocxlpmem *ocxlpmem;
> > + int minor = MINOR(devno);
> > + /*
> > +* We don't declare an RCU critical section here, as our AFU
> > +* is protected by a reference counter on the device. By the
> > time the
> > +* minor number of a device is removed from the idr, the ref
> > count of
> > +* the device is already at 0, so no user API will access that
> > AFU and
> > +* this function can't return it.
> > +*/
>
> I fixed something related in the ocxl driver (which had enough
> changes
> with the introduction of the "info" device to make a similar comment
> become wrong). See commit a58d37bce0d21. The issue is handling a
> simultaneous open() and removal of the device through /sysfs as best
> we can.
>
> We are on a file open path and it's not like we're going to have a
> thousand clients, so performance is not that critical. We can take
> the
> mutex before searching in the IDR and release it after we increment
> the
> reference count on the device.
> But that's not enough: we could still find the device in the IDR
> while
> it is being removed in free_ocxlpmem(). I believe the only safe way
> to
> address it is by removing the user-facing APIs (the char device)
> before
> calling device_unregister(). So that it's not possible to find the
> device in file_open() if it's in the middle of being removed.
>
>Fred
>
>
Ok, I'll replicate that patch & follow your advice.
> > + ocxlpmem = idr_find(&minors_idr, minor);
> > + if (ocxlpmem)
> > + ocxlpmem_get(ocxlpmem);
> > + return ocxlpmem;
> > +}
> > +
> > +static int file_open(struct inode *inode, struct file *file)
> > +{
> > + struct ocxlpmem *ocxlpmem;
> > +
> > + ocxlpmem = find_and_get_ocxlpmem(inode->i_rdev);
> > + if (!ocxlpmem)
> > + return -ENODEV;
> > +
> > + file->private_data = ocxlpmem;
> > + return 0;
> > +}
> > +
> > +static int file_release(struct inode *inode, struct file *file)
> > +{
> > + struct ocxlpmem *ocxlpmem = file->private_data;
> > +
> > + ocxlpmem_put(ocxlpmem);
> > + return 0;
> > +}
> > +
> > +static const struct file_operations fops = {
> > + .owner = THIS_MODULE,
> > + .open = file_open,
> > + .release= file_release,
> > +};
> > +
> > +/**
> > + * create_cdev() - Create the chardev in /dev for the device
> > + * @ocxlpmem: the SCM metadata
> > + * Return: 0 on suc
Re: [PATCH v3 20/27] powerpc/powernv/pmem: Forward events to userspace
On Tue, 2020-03-03 at 18:02 +1100, Andrew Donnellan wrote:
> On 21/2/20 2:27 pm, Alastair D'Silva wrote:> @@ -938,6 +955,51 @@
> static
> int ioctl_controller_stats(struct ocxlpmem *ocxlpmem,
> > return rc;
> > }
> >
> > +static int ioctl_eventfd(struct ocxlpmem *ocxlpmem,
> > +struct ioctl_ocxl_pmem_eventfd __user *uarg)
> > +{
> > + struct ioctl_ocxl_pmem_eventfd args;
> > +
> > + if (copy_from_user(&args, uarg, sizeof(args)))
> > + return -EFAULT;
> > +
> > + if (ocxlpmem->ev_ctx)
> > + return -EINVAL;
>
> I think EBUSY is more appropriate here.
>
Ok
> > +
> > + ocxlpmem->ev_ctx = eventfd_ctx_fdget(args.eventfd);
> > + if (!ocxlpmem->ev_ctx)
> > + return -EFAULT;
> > +
> > + return 0;
> > +}
> > +
> > +static int ioctl_event_check(struct ocxlpmem *ocxlpmem, u64 __user
> > *uarg)
> > +{
> > + u64 val = 0;
> > + int rc;
> > + u64 chi = 0;
> > +
> > + rc = ocxlpmem_chi(ocxlpmem, &chi);
> > + if (rc < 0)
> > + return rc;
> > +
> > + if (chi & GLOBAL_MMIO_CHI_ELA)
> > + val |= IOCTL_OCXL_PMEM_EVENT_ERROR_LOG_AVAILABLE;
> > +
> > + if (chi & GLOBAL_MMIO_CHI_CDA)
> > + val |= IOCTL_OCXL_PMEM_EVENT_CONTROLLER_DUMP_AVAILABLE;
> > +
> > + if (chi & GLOBAL_MMIO_CHI_CFFS)
> > + val |= IOCTL_OCXL_PMEM_EVENT_FIRMWARE_FATAL;
> > +
> > + if (chi & GLOBAL_MMIO_CHI_CHFS)
> > + val |= IOCTL_OCXL_PMEM_EVENT_HARDWARE_FATAL;
> > +
> > + rc = copy_to_user((u64 __user *) uarg, &val, sizeof(val));
> > +
> > + return rc;
> > +}
> > +
> > static long file_ioctl(struct file *file, unsigned int cmd,
> > unsigned long args)
> > {
> > struct ocxlpmem *ocxlpmem = file->private_data;
> > @@ -966,6 +1028,15 @@ static long file_ioctl(struct file *file,
> > unsigned int cmd, unsigned long args)
> > rc = ioctl_controller_stats(ocxlpmem,
> > (struct
> > ioctl_ocxl_pmem_controller_stats __user *)args);
> > break;
> > +
> > + case IOCTL_OCXL_PMEM_EVENTFD:
> > + rc = ioctl_eventfd(ocxlpmem,
> > + (struct ioctl_ocxl_pmem_eventfd
> > __user *)args);
> > + break;
> > +
> > + case IOCTL_OCXL_PMEM_EVENT_CHECK:
> > + rc = ioctl_event_check(ocxlpmem, (u64 __user *)args);
> > + break;
> > }
> >
> > return rc;
> > @@ -1107,6 +1178,146 @@ static void dump_error_log(struct ocxlpmem
> > *ocxlpmem)
> > kfree(buf);
> > }
> >
> > +static irqreturn_t imn0_handler(void *private)
> > +{
> > + struct ocxlpmem *ocxlpmem = private;
> > + u64 chi = 0;
> > +
> > + (void)ocxlpmem_chi(ocxlpmem, &chi);
> > +
> > + if (chi & GLOBAL_MMIO_CHI_ELA) {
> > + dev_warn(&ocxlpmem->dev, "Error log is available\n");
> > +
> > + if (ocxlpmem->ev_ctx)
> > + eventfd_signal(ocxlpmem->ev_ctx, 1);
> > + }
> > +
> > + if (chi & GLOBAL_MMIO_CHI_CDA) {
> > + dev_warn(&ocxlpmem->dev, "Controller dump is
> > available\n");
> > +
> > + if (ocxlpmem->ev_ctx)
> > + eventfd_signal(ocxlpmem->ev_ctx, 1);
> > + }
> > +
> > +
> > + return IRQ_HANDLED;
> > +}
> > +
> > +static irqreturn_t imn1_handler(void *private)
> > +{
> > + struct ocxlpmem *ocxlpmem = private;
> > + u64 chi = 0;
> > +
> > + (void)ocxlpmem_chi(ocxlpmem, &chi);
> > +
> > + if (chi & (GLOBAL_MMIO_CHI_CFFS | GLOBAL_MMIO_CHI_CHFS)) {
> > + dev_err(&ocxlpmem->dev,
> > + "Controller status is fatal, chi=0x%llx, going
> > offline\n", chi);
> > +
> > + if (ocxlpmem->nvdimm_bus) {
> > + nvdimm_bus_unregister(ocxlpmem->nvdimm_bus);
> > + ocxlpmem->nvdimm_bus = NULL;
> > + }
> > +
> > + if (ocxlpmem->ev_ctx)
> > + eventfd_signal(ocxlpmem->ev_ctx, 1);
> > + }
> > +
> > + return IRQ_HANDLED;
> > +}
> > +
> > +
> > +/**
> > + * ocxlpmem_setup_irq() - Set up the IRQs for the OpenCAPI
> &
Re: [PATCH v3 03/27] powerpc: Map & release OpenCAPI LPC memory
On Tue, 2020-03-03 at 17:10 +1100, Andrew Donnellan wrote:
> On 21/2/20 2:26 pm, Alastair D'Silva wrote:> +#ifdef
> CONFIG_MEMORY_HOTPLUG_SPARSE
> > +u64 pnv_ocxl_platform_lpc_setup(struct pci_dev *pdev, u64 size)
> > +{
> > + struct pci_controller *hose = pci_bus_to_host(pdev->bus);
> > + struct pnv_phb *phb = hose->private_data;
> > + u32 bdfn = pci_dev_id(pdev);
> > + __be64 base_addr_be64;
> > + u64 base_addr;
> > + int rc;
> > +
> > + rc = opal_npu_mem_alloc(phb->opal_id, bdfn, size,
> > &base_addr_be64);
>
> Sparse warning:
>
> https://openpower.xyz/job/snowpatch/job/snowpatch-linux-sparse/15776//artifact/linux/report.txt
>
> I think in patch 1 we need to change a uint64_t to a __be64.
>
Ok, thanks
--
Alastair D'Silva
Open Source Developer
Linux Technology Centre, IBM Australia
mob: 0423 762 819
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
RE: [PATCH v3 15/27] powerpc/powernv/pmem: Add support for near storage commands
On Mon, 2020-03-02 at 10:42 -0800, Dan Williams wrote: > On Mon, Mar 2, 2020 at 9:59 AM Frederic Barrat > wrote: > > > > > > Le 21/02/2020 à 04:27, Alastair D'Silva a écrit : > > > From: Alastair D'Silva > > > > > > Similar to the previous patch, this adds support for near storage > > > commands. > > > > > > Signed-off-by: Alastair D'Silva > > > --- > > > > Is any of these new functions ever called? > > This is my concern as well. The libnvdimm command support is limited > to the commands that Linux will use. Other passthrough commands are > supported through a passthrough interface. However, that passthrough > interface is explicitly limited to publicly documented command sets > so > that the kernel has an opportunity to constrain and consolidate > command implementations across vendors. It will be in the patch that implements overwrite. I moved that patch out of this series, as it needs more testing, so I guess I can submit this alongside it. -- Alastair D'Silva Open Source Developer Linux Technology Centre, IBM Australia mob: 0423 762 819 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 13/27] powerpc/powernv/pmem: Read the capability registers & wait for device ready
On Mon, 2020-03-02 at 18:51 +0100, Frederic Barrat wrote:
>
> Le 21/02/2020 à 04:27, Alastair D'Silva a écrit :
> > From: Alastair D'Silva
> >
> > This patch reads timeouts & firmware version from the controller,
> > and
> > uses those timeouts to wait for the controller to report that it is
> > ready
> > before handing the memory over to libnvdimm.
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > arch/powerpc/platforms/powernv/pmem/Makefile | 2 +-
> > arch/powerpc/platforms/powernv/pmem/ocxl.c| 92
> > +++
> > .../platforms/powernv/pmem/ocxl_internal.c| 19
> > .../platforms/powernv/pmem/ocxl_internal.h| 24 +
> > 4 files changed, 136 insertions(+), 1 deletion(-)
> > create mode 100644
> > arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
> >
> > diff --git a/arch/powerpc/platforms/powernv/pmem/Makefile
> > b/arch/powerpc/platforms/powernv/pmem/Makefile
> > index 1c55c4193175..4ceda25907d4 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/Makefile
> > +++ b/arch/powerpc/platforms/powernv/pmem/Makefile
> > @@ -4,4 +4,4 @@ ccflags-$(CONFIG_PPC_WERROR)+= -Werror
> >
> > obj-$(CONFIG_OCXL_PMEM) += ocxlpmem.o
> >
> > -ocxlpmem-y := ocxl.o
> > +ocxlpmem-y := ocxl.o ocxl_internal.o
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > index 3c4eeb5dcc0f..431212c9f0cc 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > @@ -8,6 +8,7 @@
> >
> > #include
> > #include
> > +#include
> > #include
> > #include
> > #include
> > @@ -215,6 +216,36 @@ static int register_lpc_mem(struct ocxlpmem
> > *ocxlpmem)
> > return 0;
> > }
> >
> > +/**
> > + * is_usable() - Is a controller usable?
> > + * @ocxlpmem: the device metadata
> > + * @verbose: True to log errors
> > + * Return: true if the controller is usable
> > + */
> > +static bool is_usable(const struct ocxlpmem *ocxlpmem, bool
> > verbose)
> > +{
> > + u64 chi = 0;
> > + int rc = ocxlpmem_chi(ocxlpmem, &chi);
> > +
> > + if (rc < 0)
> > + return false;
> > +
> > + if (!(chi & GLOBAL_MMIO_CHI_CRDY)) {
> > + if (verbose)
> > + dev_err(&ocxlpmem->dev, "controller is not
> > ready.\n");
> > + return false;
> > + }
> > +
> > + if (!(chi & GLOBAL_MMIO_CHI_MA)) {
> > + if (verbose)
> > + dev_err(&ocxlpmem->dev,
> > + "controller does not have memory
> > available.\n");
> > + return false;
> > + }
> > +
> > + return true;
> > +}
> > +
> > /**
> >* allocate_minor() - Allocate a minor number to use for an
> > OpenCAPI pmem device
> >* @ocxlpmem: the device metadata
> > @@ -328,6 +359,48 @@ static void ocxlpmem_remove(struct pci_dev
> > *pdev)
> > }
> > }
> >
> > +/**
> > + * read_device_metadata() - Retrieve config information from the
> > AFU and save it for future use
> > + * @ocxlpmem: the device metadata
> > + * Return: 0 on success, negative on failure
> > + */
> > +static int read_device_metadata(struct ocxlpmem *ocxlpmem)
> > +{
> > + u64 val;
> > + int rc;
> > +
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > GLOBAL_MMIO_CCAP0,
> > +OCXL_LITTLE_ENDIAN, &val);
> > + if (rc)
> > + return rc;
> > +
> > + ocxlpmem->scm_revision = val & 0x;
> > + ocxlpmem->read_latency = (val >> 32) & 0xFF;
> > + ocxlpmem->readiness_timeout = (val >> 48) & 0x0F;
> > + ocxlpmem->memory_available_timeout = val >> 52;
> > +
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > GLOBAL_MMIO_CCAP1,
> > +OCXL_LITTLE_ENDIAN, &val);
> > + if (rc)
> > + return rc;
> > +
> > + ocxlpmem->max_controller_dump_size = val & 0x;
> > +
> > + // Extract firmware version text
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > GLOBAL_MMIO_FWVER,
> > +OCXL_HOST_ENDIAN, (u64 *)ocxlpmem-
> > >
Re: [PATCH v3 26/27] powerpc/powernv/pmem: Expose the firmware version in sysfs
On Mon, 2020-03-02 at 18:35 +1100, Andrew Donnellan wrote: > On 21/2/20 2:27 pm, Alastair D'Silva wrote: > > From: Alastair D'Silva > > > > This information will be used by ndctl in userspace to help users > > identify > > the device. > > You should include the information from the subject line in the body > of > the commit message too. > > I think this patch could probably be squashed in with the last one. > Ok -- Alastair D'Silva Open Source Developer Linux Technology Centre, IBM Australia mob: 0423 762 819 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 16/27] powerpc/powernv/pmem: Register a character device for userspace to interact with
On Mon, 2020-03-02 at 16:34 +1100, Andrew Donnellan wrote:
> On 21/2/20 2:27 pm, Alastair D'Silva wrote:
> > From: Alastair D'Silva
> >
> > This patch introduces a character device (/dev/ocxl-scmX) which
> > further
> > patches will use to interact with userspace.
>
> As with the comments on other patches in this series, this commit
> message is lacking in explanation. What's the purpose of this device?
>
I'll reword this for v4.
> > Signed-off-by: Alastair D'Silva
> > ---
> > arch/powerpc/platforms/powernv/pmem/ocxl.c| 116
> > +-
> > .../platforms/powernv/pmem/ocxl_internal.h| 2 +
> > 2 files changed, 116 insertions(+), 2 deletions(-)
> >
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > index b8bd7e703b19..63109a870d2c 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > @@ -10,6 +10,7 @@
> > #include
> > #include
> > #include
> > +#include
> > #include
> > #include
> > #include "ocxl_internal.h"
> > @@ -339,6 +340,9 @@ static void free_ocxlpmem(struct ocxlpmem
> > *ocxlpmem)
> >
> > free_minor(ocxlpmem);
> >
> > + if (ocxlpmem->cdev.owner)
> > + cdev_del(&ocxlpmem->cdev);
> > +
> > if (ocxlpmem->metadata_addr)
> > devm_memunmap(&ocxlpmem->dev, ocxlpmem->metadata_addr);
> >
> > @@ -396,6 +400,70 @@ static int ocxlpmem_register(struct ocxlpmem
> > *ocxlpmem)
> > return device_register(&ocxlpmem->dev);
> > }
> >
> > +static void ocxlpmem_put(struct ocxlpmem *ocxlpmem)
> > +{
> > + put_device(&ocxlpmem->dev);
> > +}
> > +
> > +static struct ocxlpmem *ocxlpmem_get(struct ocxlpmem *ocxlpmem)
> > +{
> > + return (get_device(&ocxlpmem->dev) == NULL) ? NULL : ocxlpmem;
> > +}
> > +
> > +static struct ocxlpmem *find_and_get_ocxlpmem(dev_t devno)
> > +{
> > + struct ocxlpmem *ocxlpmem;
> > + int minor = MINOR(devno);
> > + /*
> > +* We don't declare an RCU critical section here, as our AFU
> > +* is protected by a re0ference counter on the device. By the
> > time the
> > +* minor number of a device is removed from the idr, the ref
> > count of
> > +* the device is already at 0, so no user API will access that
> > AFU and
> > +* this function can't return it.
> > +*/
> > + ocxlpmem = idr_find(&minors_idr, minor);
> > + if (ocxlpmem)
> > + ocxlpmem_get(ocxlpmem);
> > + return ocxlpmem;
> > +}
> > +
> > +static int file_open(struct inode *inode, struct file *file)
> > +{
> > + struct ocxlpmem *ocxlpmem;
> > +
> > + ocxlpmem = find_and_get_ocxlpmem(inode->i_rdev);
> > + if (!ocxlpmem)
> > + return -ENODEV;
> > +
> > + file->private_data = ocxlpmem;
> > + return 0;
> > +}
> > +
> > +static int file_release(struct inode *inode, struct file *file)
> > +{
> > + struct ocxlpmem *ocxlpmem = file->private_data;
> > +
> > + ocxlpmem_put(ocxlpmem);
> > + return 0;
> > +}
> > +
> > +static const struct file_operations fops = {
> > + .owner = THIS_MODULE,
> > + .open = file_open,
> > + .release= file_release,
> > +};
> > +
> > +/**
> > + * create_cdev() - Create the chardev in /dev for the device
> > + * @ocxlpmem: the SCM metadata
> > + * Return: 0 on success, negative on failure
> > + */
> > +static int create_cdev(struct ocxlpmem *ocxlpmem)
> > +{
> > + cdev_init(&ocxlpmem->cdev, &fops);
> > + return cdev_add(&ocxlpmem->cdev, ocxlpmem->dev.devt, 1);
> > +}
> > +
> > /**
> >* ocxlpmem_remove() - Free an OpenCAPI persistent memory device
> >* @pdev: the PCI device information struct
> > @@ -572,6 +640,11 @@ static int probe(struct pci_dev *pdev, const
> > struct pci_device_id *ent)
> > goto err;
> > }
> >
> > + if (create_cdev(ocxlpmem)) {
> > + dev_err(&pdev->dev, "Could not create character
> > device\n");
> > + goto err;
> > + }
> > +
> > elapsed = 0;
> > timeout = ocxlpmem->readiness_timeout + ocxlpmem-
> > >memory_available_
Re: [PATCH v3 21/27] powerpc/powernv/pmem: Add an IOCTL to request controller health & perf data
On Fri, 2020-02-28 at 17:12 +1100, Andrew Donnellan wrote: > On 21/2/20 2:27 pm, Alastair D'Silva wrote: > > From: Alastair D'Silva > > > > When health & performance data is requested from the controller, > > it responds with an error log containing the requested information. > > > > This patch allows the request to me issued via an IOCTL. > > A better explanation would be good - this IOCTL triggers a request > to > the controller to collect controller health/perf data, and the > controller will later respond with an error log that can be picked > up > via the error log IOCTL that you've defined earlier. > > Ok -- Alastair D'Silva Open Source Developer Linux Technology Centre, IBM Australia mob: 0423 762 819 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
RE: [PATCH v3 25/27] powerpc/powernv/pmem: Expose the serial number in sysfs
On Mon, 2020-03-02 at 10:42 +1100, Alastair D'Silva wrote:
> On Fri, 2020-02-28 at 08:15 +0100, Greg Kroah-Hartman wrote:
> > On Fri, Feb 28, 2020 at 05:25:31PM +1100, Andrew Donnellan wrote:
> > > On 21/2/20 2:27 pm, Alastair D'Silva wrote:
> > > > +int ocxlpmem_sysfs_add(struct ocxlpmem *ocxlpmem)
> > > > +{
> > > > + int i, rc;
> > > > +
> > > > + for (i = 0; i < ARRAY_SIZE(attrs); i++) {
> > > > + rc = device_create_file(&ocxlpmem->dev,
> > > > &attrs[i]);
> > > > + if (rc) {
> > > > + for (; --i >= 0;)
> > > > + device_remove_file(&ocxlpmem-
> > > > >dev,
> > > > &attrs[i]);
> > >
> > > I'd rather avoid weird for loop constructs if possible.
> > >
> > > Is it actually dangerous to call device_remove_file() on an attr
> > > that hasn't
> > > been added? If not then I'd rather define an err: label and loop
> > > over the
> > > whole array there.
> >
> > None of this should be used at all, just use attribute groups
> > properly
> > and the driver core will handle this all for you.
> >
> > device_create/remove_file should never be called by anyone anymore
> > if
> > at all
> > possible.
> >
> > thanks,
> >
> > greg k-h
>
> Thanks, I'll rework it to use the .groups member of struct
> pci_driver.
>
I ended up making these available as DIMM attributes instead.
--
Alastair D'Silva
Open Source Developer
Linux Technology Centre, IBM Australia
mob: 0423 762 819
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
RE: [PATCH v3 25/27] powerpc/powernv/pmem: Expose the serial number in sysfs
On Fri, 2020-02-28 at 08:15 +0100, Greg Kroah-Hartman wrote:
> On Fri, Feb 28, 2020 at 05:25:31PM +1100, Andrew Donnellan wrote:
> > On 21/2/20 2:27 pm, Alastair D'Silva wrote:
> > > +int ocxlpmem_sysfs_add(struct ocxlpmem *ocxlpmem)
> > > +{
> > > + int i, rc;
> > > +
> > > + for (i = 0; i < ARRAY_SIZE(attrs); i++) {
> > > + rc = device_create_file(&ocxlpmem->dev, &attrs[i]);
> > > + if (rc) {
> > > + for (; --i >= 0;)
> > > + device_remove_file(&ocxlpmem->dev,
> > > &attrs[i]);
> >
> > I'd rather avoid weird for loop constructs if possible.
> >
> > Is it actually dangerous to call device_remove_file() on an attr
> > that hasn't
> > been added? If not then I'd rather define an err: label and loop
> > over the
> > whole array there.
>
> None of this should be used at all, just use attribute groups
> properly
> and the driver core will handle this all for you.
>
> device_create/remove_file should never be called by anyone anymore if
> at all
> possible.
>
> thanks,
>
> greg k-h
Thanks, I'll rework it to use the .groups member of struct pci_driver.
--
Alastair D'Silva
Open Source Developer
Linux Technology Centre, IBM Australia
mob: 0423 762 819
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 10/27] powerpc: Add driver for OpenCAPI Persistent Memory
On Thu, 2020-02-27 at 21:44 +0100, Frederic Barrat wrote: > > Le 21/02/2020 à 04:27, Alastair D'Silva a écrit : > > From: Alastair D'Silva > > > > This driver exposes LPC memory on OpenCAPI pmem cards > > as an NVDIMM, allowing the existing nvram infrastructure > > to be used. > > > > Namespace metadata is stored on the media itself, so > > scm_reserve_metadata() maps 1 section's worth of PMEM storage > > at the start to hold this. The rest of the PMEM range is registered > > with libnvdimm as an nvdimm. scm_ndctl_config_read/write/size() > > provide > > callbacks to libnvdimm to access the metadata. > > > > Signed-off-by: Alastair D'Silva > > --- > > arch/powerpc/platforms/powernv/Kconfig| 3 + > > arch/powerpc/platforms/powernv/Makefile | 1 + > > arch/powerpc/platforms/powernv/pmem/Kconfig | 15 + > > arch/powerpc/platforms/powernv/pmem/Makefile | 7 + > > arch/powerpc/platforms/powernv/pmem/ocxl.c| 473 > > ++ > > .../platforms/powernv/pmem/ocxl_internal.h| 28 ++ > > 6 files changed, 527 insertions(+) > > create mode 100644 arch/powerpc/platforms/powernv/pmem/Kconfig > > create mode 100644 arch/powerpc/platforms/powernv/pmem/Makefile > > create mode 100644 arch/powerpc/platforms/powernv/pmem/ocxl.c > > create mode 100644 > > arch/powerpc/platforms/powernv/pmem/ocxl_internal.h > > > > diff --git a/arch/powerpc/platforms/powernv/Kconfig > > b/arch/powerpc/platforms/powernv/Kconfig > > index 938803eab0ad..fc8976af0e52 100644 > > --- a/arch/powerpc/platforms/powernv/Kconfig > > +++ b/arch/powerpc/platforms/powernv/Kconfig > > @@ -50,3 +50,6 @@ config PPC_VAS > > config SCOM_DEBUGFS > > bool "Expose SCOM controllers via debugfs" > > depends on DEBUG_FS > > + > > +source "arch/powerpc/platforms/powernv/pmem/Kconfig" > > + > > diff --git a/arch/powerpc/platforms/powernv/Makefile > > b/arch/powerpc/platforms/powernv/Makefile > > index c0f8120045c3..0bbd72988b6f 100644 > > --- a/arch/powerpc/platforms/powernv/Makefile > > +++ b/arch/powerpc/platforms/powernv/Makefile > > @@ -21,3 +21,4 @@ obj-$(CONFIG_PPC_VAS) += vas.o vas-window.o > > vas-debug.o > > obj-$(CONFIG_OCXL_BASE) += ocxl.o > > obj-$(CONFIG_SCOM_DEBUGFS) += opal-xscom.o > > obj-$(CONFIG_PPC_SECURE_BOOT) += opal-secvar.o > > +obj-$(CONFIG_LIBNVDIMM) += pmem/ > > diff --git a/arch/powerpc/platforms/powernv/pmem/Kconfig > > b/arch/powerpc/platforms/powernv/pmem/Kconfig > > new file mode 100644 > > index ..c5d927520920 > > --- /dev/null > > +++ b/arch/powerpc/platforms/powernv/pmem/Kconfig > > @@ -0,0 +1,15 @@ > > +# SPDX-License-Identifier: GPL-2.0-only > > +if LIBNVDIMM > > + > > +config OCXL_PMEM > > + tristate "OpenCAPI Persistent Memory" > > + depends on LIBNVDIMM && PPC_POWERNV && PCI && EEH && > > ZONE_DEVICE && OCXL > > + help > > + Exposes devices that implement the OpenCAPI Storage Class > > Memory > > + specification as persistent memory regions. You may also want > > + DEV_DAX, DEV_DAX_PMEM & FS_DAX if you plan on using DAX > > devices > > + stacked on top of this driver. > > + > > + Select N if unsure. > > + > > +endif > > diff --git a/arch/powerpc/platforms/powernv/pmem/Makefile > > b/arch/powerpc/platforms/powernv/pmem/Makefile > > new file mode 100644 > > index ..1c55c4193175 > > --- /dev/null > > +++ b/arch/powerpc/platforms/powernv/pmem/Makefile > > @@ -0,0 +1,7 @@ > > +# SPDX-License-Identifier: GPL-2.0 > > + > > +ccflags-$(CONFIG_PPC_WERROR) += -Werror > > + > > +obj-$(CONFIG_OCXL_PMEM) += ocxlpmem.o > > + > > +ocxlpmem-y := ocxl.o > > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c > > b/arch/powerpc/platforms/powernv/pmem/ocxl.c > > new file mode 100644 > > index ..3c4eeb5dcc0f > > --- /dev/null > > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c > > @@ -0,0 +1,473 @@ > > +// SPDX-License-Id > > +// Copyright 2019 IBM Corp. > > + > > +/* > > + * A driver for OpenCAPI devices that implement the Storage Class > > + * Memory specification. > > + */ > > + > > +#include > > +#include > > +#include > > +#include > > +#include > > +#include "ocxl_internal.h" > > + > > + > &g
RE: [PATCH v3 14/27] powerpc/powernv/pmem: Add support for Admin commands
On Thu, 2020-02-27 at 09:01 -0800, Dan Williams wrote: > On Thu, Feb 20, 2020 at 7:28 PM Alastair D'Silva < > alast...@au1.ibm.com> wrote: > > From: Alastair D'Silva > > > > This patch requests the metadata required to issue admin commands, > > as well > > as some helper functions to construct and check the completion of > > the > > commands. > > What are the admin commands? Any pointer to a spec? Why does Linux > need to support these commands? I'll flesh these out for the next spin, thanks. -- Alastair D'Silva Open Source Developer Linux Technology Centre, IBM Australia mob: 0423 762 819 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 15/27] powerpc/powernv/pmem: Add support for near storage commands
On Thu, 2020-02-27 at 19:30 +1100, Andrew Donnellan wrote:
> On 21/2/20 2:27 pm, Alastair D'Silva wrote:> +int
> ns_response_handled(const struct ocxlpmem *ocxlpmem)
> > +{
> > + return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu,
> > GLOBAL_MMIO_CHIC,
> > + OCXL_LITTLE_ENDIAN,
> > GLOBAL_MMIO_CHI_NSCRA);
> > +}
>
> Same comment as on the last patch - I think we're meant to be
> clearing
> this bit, not setting it to 1,
>
Same reply :) Writing to the CHIC register clears the bit in CHI.
>
--
Alastair D'Silva
Open Source Developer
Linux Technology Centre, IBM Australia
mob: 0423 762 819
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 14/27] powerpc/powernv/pmem: Add support for Admin commands
On Thu, 2020-02-27 at 19:27 +1100, Andrew Donnellan wrote:
> On 27/2/20 7:22 pm, Andrew Donnellan wrote:
> > > +int admin_command_request(struct ocxlpmem *ocxlpmem, u8 op_code)
> > > +{
> > > +u64 val;
> > > +int rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > > GLOBAL_MMIO_CHI,
> > > + OCXL_LITTLE_ENDIAN, &val);
> > > +if (rc)
> > > +return rc;
> >
> > Ignoring the value here expected, you're just trying to verify that
> > you
> > don't see an error on the read?
>
> I see that in the next patch, in ns_command_request() you check that
> NSCRA is 1 - did you mean to check that ACRA = 1 here?
>
>
I was in one version, but that was causing problems in startup since
there was successful prior command to assert ACRA.
I should remove the NSCRA check too.
--
Alastair D'Silva
Open Source Developer
Linux Technology Centre, IBM Australia
mob: 0423 762 819
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 14/27] powerpc/powernv/pmem: Add support for Admin commands
On Thu, 2020-02-27 at 19:22 +1100, Andrew Donnellan wrote:
> On 21/2/20 2:27 pm, Alastair D'Silva wrote:
> > From: Alastair D'Silva
> >
> > This patch requests the metadata required to issue admin commands,
> > as well
> > as some helper functions to construct and check the completion of
> > the
> > commands.
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > arch/powerpc/platforms/powernv/pmem/ocxl.c| 65
> > .../platforms/powernv/pmem/ocxl_internal.c| 153
> > ++
> > .../platforms/powernv/pmem/ocxl_internal.h| 61 +++
> > 3 files changed, 279 insertions(+)
> >
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > index 431212c9f0cc..4e782d22605b 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > @@ -216,6 +216,58 @@ static int register_lpc_mem(struct ocxlpmem
> > *ocxlpmem)
> > return 0;
> > }
> >
> > +/**
> > + * extract_command_metadata() - Extract command data from MMIO &
> > save it for further use
> > + * @ocxlpmem: the device metadata
> > + * @offset: The base address of the command data structures
> > (address of CREQO)
> > + * @command_metadata: A pointer to the command metadata to
> > populate
> > + * Return: 0 on success, negative on failure
> > + */
> > +static int extract_command_metadata(struct ocxlpmem *ocxlpmem, u32
> > offset,
> > + struct command_metadata
> > *command_metadata)
> > +{
> > + int rc;
> > + u64 tmp;
> > +
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, offset,
> > OCXL_LITTLE_ENDIAN,
> > +&tmp);
> > + if (rc)
> > + return rc;
> > +
> > + command_metadata->request_offset = tmp >> 32;
> > + command_metadata->response_offset = tmp & 0x;
> > +
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, offset + 8,
> > OCXL_LITTLE_ENDIAN,
> > +&tmp);
> > + if (rc)
> > + return rc;
> > +
> > + command_metadata->data_offset = tmp >> 32;
> > + command_metadata->data_size = tmp & 0x;
> > +
> > + command_metadata->id = 0;
> > +
> > + return 0;
> > +}
> > +
> > +/**
> > + * setup_command_metadata() - Set up the command metadata
> > + * @ocxlpmem: the device metadata
> > + */
> > +static int setup_command_metadata(struct ocxlpmem *ocxlpmem)
> > +{
> > + int rc;
> > +
> > + mutex_init(&ocxlpmem->admin_command.lock);
> > +
> > + rc = extract_command_metadata(ocxlpmem, GLOBAL_MMIO_ACMA_CREQO,
> > + &ocxlpmem->admin_command);
> > + if (rc)
> > + return rc;
> > +
> > + return 0;
> > +}
> > +
> > /**
> >* is_usable() - Is a controller usable?
> >* @ocxlpmem: the device metadata
> > @@ -456,6 +508,14 @@ static int probe(struct pci_dev *pdev, const
> > struct pci_device_id *ent)
> > }
> > ocxlpmem->pdev = pdev;
> >
> > + ocxlpmem->timeouts[ADMIN_COMMAND_ERRLOG] = 2000; // ms
> > + ocxlpmem->timeouts[ADMIN_COMMAND_HEARTBEAT] = 100; // ms
> > + ocxlpmem->timeouts[ADMIN_COMMAND_SMART] = 100; // ms
> > + ocxlpmem->timeouts[ADMIN_COMMAND_CONTROLLER_DUMP] = 1000; // ms
> > + ocxlpmem->timeouts[ADMIN_COMMAND_CONTROLLER_STATS] = 100; // ms
> > + ocxlpmem->timeouts[ADMIN_COMMAND_SHUTDOWN] = 1000; // ms
> > + ocxlpmem->timeouts[ADMIN_COMMAND_FW_UPDATE] = 16000; // ms
>
> Why are we keeping these timeouts in a per device struct? I can't
> see
> anywhere where we change these values.
>
These are overwritten in a later patch, which I've missed! thanks for
pointing this out.
These initial values will be overwritten by card specific timeouts.
> > +
> > pci_set_drvdata(pdev, ocxlpmem);
> >
> > ocxlpmem->ocxl_fn = ocxl_function_open(pdev);
> > @@ -501,6 +561,11 @@ static int probe(struct pci_dev *pdev, const
> > struct pci_device_id *ent)
> > goto err;
> > }
> >
> > + if (setup_command_metadata(ocxlpmem)) {
> > + dev_err(&pdev->dev, "Could not read OCXL command
> > matada\n");
>
> metadata
Wow, not sure how that hap
Re: [PATCH v3 12/27] powerpc/powernv/pmem: Add register addresses & status values to the header
On Thu, 2020-02-27 at 16:08 +1100, Andrew Donnellan wrote: > On 21/2/20 2:27 pm, Alastair D'Silva wrote: > > From: Alastair D'Silva > > > > These values have been taken from the device specifications. > > > > Signed-off-by: Alastair D'Silva > > I've compared these values against the internal version of the > device > specifications that I have access to, and they appear to match. > > A few minor comments below, otherwise: > > Reviewed-by: Andrew Donnellan > > > +#define GLOBAL_MMIO_HCI_ACRW BIT_ULL > > (0) > > +#define GLOBAL_MMIO_HCI_NSCRW BIT_ULL > > (1) > > +#define GLOBAL_MMIO_HCI_AFU_RESET BIT_ULL(2) > > +#define GLOBAL_MMIO_HCI_FW_DEBUG BIT_ULL(3) > > +#define GLOBAL_MMIO_HCI_CONTROLLER_DUMPBIT_ULL > > (4) > > +#define GLOBAL_MMIO_HCI_CONTROLLER_DUMP_COLLECTED BIT_ULL(5) > > +#define GLOBAL_MMIO_HCI_REQ_HEALTH_PERFBIT_ULL > > (6) > > The labelling of some of these bits deviates from the standard > abbreviations in the spec, which is fine I guess as these names are > more > descriptive, but maybe add a brief comment with the original > abbreviation? > Ok > > + > > +#define ADMIN_COMMAND_HEARTBEAT0x00u > > +#define ADMIN_COMMAND_SHUTDOWN 0x01u > > +#define ADMIN_COMMAND_FW_UPDATE0x02u > > +#define ADMIN_COMMAND_FW_DEBUG 0x03u > > +#define ADMIN_COMMAND_ERRLOG 0x04u > > +#define ADMIN_COMMAND_SMART0x05u > > +#define ADMIN_COMMAND_CONTROLLER_STATS 0x06u > > +#define ADMIN_COMMAND_CONTROLLER_DUMP 0x07u > > +#define ADMIN_COMMAND_CMD_CAPS 0x08u > > +#define ADMIN_COMMAND_MAX 0x08u > > + > > +#define STATUS_SUCCESS 0x00 > > +#define STATUS_MEM_UNAVAILABLE 0x20 > > There's also a "blocked on account of background task" code, 0x21. > Ok > > +#define STATUS_BAD_OPCODE 0x50 > > +#define STATUS_BAD_REQUEST_PARM0x51 > > +#define STATUS_BAD_DATA_PARM 0x52 > > +#define STATUS_DEBUG_BLOCKED 0x70 > > +#define STATUS_FAIL0xFF > > + > > +#define STATUS_FW_UPDATE_BLOCKED 0x21 > > +#define STATUS_FW_ARG_INVALID 0x51 > > +#define STATUS_FW_INVALID 0x52 > > These status codes seem, from the specification, to correspond to > the > generic error codes above, so perhaps they're not needed. > These will be used in warn_status_fw_update() later, but I'll alias them to make it clear that they are shadowing values -- Alastair D'Silva Open Source Developer Linux Technology Centre, IBM Australia mob: 0423 762 819 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 13/27] powerpc/powernv/pmem: Read the capability registers & wait for device ready
On Thu, 2020-02-27 at 14:54 +1100, Andrew Donnellan wrote:
> On 21/2/20 2:27 pm, Alastair D'Silva wrote:
> > +/**
> > + * read_device_metadata() - Retrieve config information from the
> > AFU and save it for future use
> > + * @ocxlpmem: the device metadata
> > + * Return: 0 on success, negative on failure
> > + */
> > +static int read_device_metadata(struct ocxlpmem *ocxlpmem)
> > +{
> > + u64 val;
> > + int rc;
> > +
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > GLOBAL_MMIO_CCAP0,
> > +OCXL_LITTLE_ENDIAN, &val);
> > + if (rc)
> > + return rc;
> > +
> > + ocxlpmem->scm_revision = val & 0x;
> > + ocxlpmem->read_latency = (val >> 32) & 0xFF;
>
> This field is 16 bits in the spec, so the mask should be 0x I
> think?
>
You're right, I'll fix it.
> Maybe we should generalise the EXTRACT_BITS macro we use in ocxl :)
>
--
Alastair D'Silva
Open Source Developer
Linux Technology Centre, IBM Australia
mob: 0423 762 819
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
RE: [PATCH v3 04/27] ocxl: Remove unnecessary externs
> -Original Message- > From: Baoquan He > Sent: Wednesday, 26 February 2020 7:15 PM > To: Alastair D'Silva > Cc: alast...@d-silva.org; Aneesh Kumar K . V > ; Oliver O'Halloran ; > Benjamin Herrenschmidt ; Paul Mackerras > ; Michael Ellerman ; Frederic > Barrat ; Andrew Donnellan ; > Arnd Bergmann ; Greg Kroah-Hartman > ; Dan Williams ; > Vishal Verma ; Dave Jiang > ; Ira Weiny ; Andrew Morton > ; Mauro Carvalho Chehab > ; David S. Miller ; > Rob Herring ; Anton Blanchard ; > Krzysztof Kozlowski ; Mahesh Salgaonkar > ; Madhavan Srinivasan > ; Cédric Le Goater ; Anju T > Sudhakar ; Hari Bathini > ; Thomas Gleixner ; Greg > Kurz ; Nicholas Piggin ; Masahiro > Yamada ; Alexey Kardashevskiy > ; linux-ker...@vger.kernel.org; linuxppc- > d...@lists.ozlabs.org; linux-nvdimm@lists.01.org; linux...@kvack.org > Subject: Re: [PATCH v3 04/27] ocxl: Remove unnecessary externs > > On 02/21/20 at 02:26pm, Alastair D'Silva wrote: > > From: Alastair D'Silva > > > > Function declarations don't need externs, remove the existing ones so > > they are consistent with newer code > > > > Signed-off-by: Alastair D'Silva > > --- > > arch/powerpc/include/asm/pnv-ocxl.h | 32 ++--- > > include/misc/ocxl.h | 6 +++--- > > 2 files changed, 18 insertions(+), 20 deletions(-) > > > > diff --git a/arch/powerpc/include/asm/pnv-ocxl.h > > b/arch/powerpc/include/asm/pnv-ocxl.h > > index 0b2a6707e555..b23c99bc0c84 100644 > > --- a/arch/powerpc/include/asm/pnv-ocxl.h > > +++ b/arch/powerpc/include/asm/pnv-ocxl.h > > @@ -9,29 +9,27 @@ > > #define PNV_OCXL_TL_BITS_PER_RATE 4 > > #define PNV_OCXL_TL_RATE_BUF_SIZE > ((PNV_OCXL_TL_MAX_TEMPLATE+1) * PNV_OCXL_TL_BITS_PER_RATE / 8) > > > > -extern int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 > *enabled, > > - u16 *supported); > > It works w or w/o extern when declare functions. Searching 'extern' > under include can find so many functions with 'extern' adding. Do we have a > explicit standard if we should add or remove 'exter' in function declaration? > > I have no objection to this patch, just want to make clear so that I can handle > it w/o confusion. > > Thanks > Baoquan > For the OpenCAPI driver, we have settled on not having 'extern' on functions. I don't think I've seen a standard that supports or refutes this, but it does not value add. -- Alastair D'Silva mob: 0423 762 819 skype: alastair_dsilva msn: alast...@d-silva.org blog: http://alastair.d-silva.orgTwitter: @EvilDeece ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 10/27] powerpc: Add driver for OpenCAPI Persistent Memory
On Wed, 2020-02-26 at 16:07 +1100, Andrew Donnellan wrote:
> On 21/2/20 2:27 pm, Alastair D'Silva wrote:
> > From: Alastair D'Silva
> >
> > This driver exposes LPC memory on OpenCAPI pmem cards
> > as an NVDIMM, allowing the existing nvram infrastructure
> > to be used.
> >
> > Namespace metadata is stored on the media itself, so
> > scm_reserve_metadata() maps 1 section's worth of PMEM storage
> > at the start to hold this. The rest of the PMEM range is registered
> > with libnvdimm as an nvdimm. scm_ndctl_config_read/write/size()
> > provide
> > callbacks to libnvdimm to access the metadata.
> >
> > Signed-off-by: Alastair D'Silva
>
> I'm not particularly familiar with the nvdimm subsystem, so the scope
> of
> my review is more on the ocxl + misc issues side.
>
> A few minor checkpatch warnings that don't matter all that much:
>
> https://openpower.xyz/job/snowpatch/job/snowpatch-linux-checkpatch/11786//artifact/linux/checkpatch.log
>
> A few other comments below.
>
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > new file mode 100644
> > index ..3c4eeb5dcc0f
> > --- /dev/null
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > @@ -0,0 +1,473 @@
> > +// SPDX-License-Id
> > +// Copyright 2019 IBM Corp.
> > +
> > +/*
> > + * A driver for OpenCAPI devices that implement the Storage Class
> > + * Memory specification.
> > + */
> > +
> > +#include
> > +#include
> > +#include
> > +#include
> > +#include
> > +#include "ocxl_internal.h"
> > +
> > +
> > +static const struct pci_device_id ocxlpmem_pci_tbl[] = {
> > + { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0625), },
> > + { }
> > +};
> > +
> > +MODULE_DEVICE_TABLE(pci, ocxlpmem_pci_tbl);
> > +
> > +#define NUM_MINORS 256 // Total to reserve
> > +
> > +static dev_t ocxlpmem_dev;
> > +static struct class *ocxlpmem_class;
> > +static struct mutex minors_idr_lock;
> > +static struct idr minors_idr;
> > +
> > +/**
> > + * ndctl_config_write() - Handle a ND_CMD_SET_CONFIG_DATA command
> > from ndctl
> > + * @ocxlpmem: the device metadata
> > + * @command: the incoming data to write
> > + * Return: 0 on success, negative on failure
> > + */
> > +static int ndctl_config_write(struct ocxlpmem *ocxlpmem,
> > + struct nd_cmd_set_config_hdr *command)
> > +{
> > + if (command->in_offset + command->in_length > LABEL_AREA_SIZE)
> > + return -EINVAL;
> > +
> > + memcpy_flushcache(ocxlpmem->metadata_addr + command->in_offset,
> > command->in_buf,
> > + command->in_length);
>
> Out of scope for this patch - given that we use memcpy_mcsafe in the
> config read, does it make sense to change memcpy_flushcache to be
> mcsafe
> as well?
>
Aneesh has confirmed that stores don't generate machine checks.
> > +
> > + return 0;
> > +}
> > +
> > +/**
> > + * ndctl_config_read() - Handle a ND_CMD_GET_CONFIG_DATA command
> > from ndctl
> > + * @ocxlpmem: the device metadata
> > + * @command: the read request
> > + * Return: 0 on success, negative on failure
> > + */
> > +static int ndctl_config_read(struct ocxlpmem *ocxlpmem,
> > +struct nd_cmd_get_config_data_hdr
> > *command)
> > +{
> > + if (command->in_offset + command->in_length > LABEL_AREA_SIZE)
> > + return -EINVAL;
> > +
> > + memcpy_mcsafe(command->out_buf, ocxlpmem->metadata_addr +
> > command->in_offset,
> > + command->in_length);
> > +
> > + return 0;
> > +}
> > +
> > +/**
> > + * ndctl_config_size() - Handle a ND_CMD_GET_CONFIG_SIZE command
> > from ndctl
> > + * @command: the read request
> > + * Return: 0 on success, negative on failure
> > + */
> > +static int ndctl_config_size(struct nd_cmd_get_config_size
> > *command)
> > +{
> > + command->status = 0;
> > + command->config_size = LABEL_AREA_SIZE;
> > + command->max_xfer = PAGE_SIZE;
> > +
> > + return 0;
> > +}
> > +
> > +static int ndctl(struct nvdimm_bus_descriptor *nd_desc,
> > +struct nvdimm *nvdimm,
> > +unsigned int cmd, void *buf, unsigned int buf_len, int
> > *cmd_rc)
> > +{
>
RE: [PATCH v3 00/27] Add support for OpenCAPI Persistent Memory devices
On Tue, 2020-02-25 at 16:32 -0800, Dan Williams wrote:
> On Tue, Feb 25, 2020 at 4:14 PM Alastair D'Silva <
> alast...@au1.ibm.com> wrote:
> > On Mon, 2020-02-24 at 17:51 +1100, Oliver O'Halloran wrote:
> > > On Mon, Feb 24, 2020 at 3:43 PM Alastair D'Silva <
> > > alast...@au1.ibm.com> wrote:
> > > > On Sun, 2020-02-23 at 20:37 -0800, Matthew Wilcox wrote:
> > > > > On Mon, Feb 24, 2020 at 03:34:07PM +1100, Alastair D'Silva
> > > > > wrote:
> > > > > > V3:
> > > > > > - Rebase against next/next-20200220
> > > > > > - Move driver to arch/powerpc/platforms/powernv, we now
> > > > > > expect
> > > > > > this
> > > > > > driver to go upstream via the powerpc tree
> > > > >
> > > > > That's rather the opposite direction of normal; mostly
> > > > > drivers
> > > > > live
> > > > > under
> > > > > drivers/ and not in arch/. It's easier for drivers to get
> > > > > overlooked
> > > > > when doing tree-wide changes if they're hiding.
> > > >
> > > > This is true, however, given that it was not all that desirable
> > > > to
> > > > have
> > > > it under drivers/nvdimm, it's sister driver (for the same
> > > > hardware)
> > > > is
> > > > also under arch, and that we don't expect this driver to be
> > > > used on
> > > > any
> > > > platform other than powernv, we think this was the most
> > > > reasonable
> > > > place to put it.
> > >
> > > Historically powernv specific platform drivers go in their
> > > respective
> > > subsystem trees rather than in arch/ and I'd prefer we kept it
> > > that
> > > way. When I added the papr_scm driver I put it in the pseries
> > > platform
> > > directory because most of the pseries paravirt code lives there
> > > for
> > > some reason; I don't know why. Luckily for me that followed the
> > > same
> > > model that Dan used when he put the NFIT driver in drivers/acpi/
> > > and
> > > the libnvdimm core in drivers/nvdimm/ so we didn't have anything
> > > to
> > > argue about. However, as Matthew pointed out, it is at odds with
> > > how
> > > most subsystems operate. Is there any particular reason we're
> > > doing
> > > things this way or should we think about moving libnvdimm users
> > > to
> > > drivers/nvdimm/?
> > >
> > > Oliver
> >
> > I'm not too fussed where it ends up, as long as it ends up
> > somewhere :)
> >
> > From what I can tell, the issue is that we have both
> > "infrastructure"
> > drivers, and end-device drivers. To me, it feels like
> > drivers/nvdimm
> > should contain both, and I think this feels like the right
> > approach.
> >
> > I could move it back to drivers/nvdimm/ocxl, but I felt that it was
> > only tolerated there, not desired. This could be cleared up with a
> > response from Dan Williams, and if it is indeed dersired, this is
> > my
> > preferred location.
>
> Apologies if I gave the impression it was only tolerated. I'm ok with
> drivers/nvdimm/ocxl/, and to the larger point I'd also be ok with a
> drivers/{acpi => nvdimm}/nfit and {arch/powerpc/platforms/pseries =>
> drivers/nvdimm}/papr_scm.c move as well to keep all the consumers of
> the nvdimm related code together with the core.
Great, thanks for clarifying, text is so imprecise when it comes to
nuance :)
I'll move ti back to drivers/nvdimm/ocxl then.
--
Alastair D'Silva
Open Source Developer
Linux Technology Centre, IBM Australia
mob: 0423 762 819
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 06/27] ocxl: Tally up the LPC memory on a link & allow it to be mapped
On Tue, 2020-02-25 at 17:30 +0100, Frederic Barrat wrote:
>
> Le 21/02/2020 à 04:26, Alastair D'Silva a écrit :
> > From: Alastair D'Silva
> >
> > Tally up the LPC memory on an OpenCAPI link & allow it to be mapped
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > drivers/misc/ocxl/core.c | 10 ++
> > drivers/misc/ocxl/link.c | 53
> > +++
> > drivers/misc/ocxl/ocxl_internal.h | 33 +++
> > 3 files changed, 96 insertions(+)
> >
> > diff --git a/drivers/misc/ocxl/core.c b/drivers/misc/ocxl/core.c
> > index b7a09b21ab36..2531c6cf19a0 100644
> > --- a/drivers/misc/ocxl/core.c
> > +++ b/drivers/misc/ocxl/core.c
> > @@ -230,8 +230,18 @@ static int configure_afu(struct ocxl_afu *afu,
> > u8 afu_idx, struct pci_dev *dev)
> > if (rc)
> > goto err_free_pasid;
> >
> > + if (afu->config.lpc_mem_size || afu-
> > >config.special_purpose_mem_size) {
> > + rc = ocxl_link_add_lpc_mem(afu->fn->link, afu-
> > >config.lpc_mem_offset,
> > + afu->config.lpc_mem_size +
> > + afu-
> > >config.special_purpose_mem_size);
> > + if (rc)
> > + goto err_free_mmio;
> > + }
> > +
> > return 0;
> >
> > +err_free_mmio:
> > + unmap_mmio_areas(afu);
> > err_free_pasid:
> > reclaim_afu_pasid(afu);
> > err_free_actag:
> > diff --git a/drivers/misc/ocxl/link.c b/drivers/misc/ocxl/link.c
> > index 58d111afd9f6..1e039cc5ebe5 100644
> > --- a/drivers/misc/ocxl/link.c
> > +++ b/drivers/misc/ocxl/link.c
> > @@ -84,6 +84,11 @@ struct ocxl_link {
> > int dev;
> > atomic_t irq_available;
> > struct spa *spa;
> > + struct mutex lpc_mem_lock; /* protects lpc_mem & lpc_mem_sz */
> > + u64 lpc_mem_sz; /* Total amount of LPC memory presented on the
> > link */
> > + u64 lpc_mem;
> > + int lpc_consumers;
> > +
> > void *platform_data;
> > };
> > static struct list_head links_list = LIST_HEAD_INIT(links_list);
> > @@ -396,6 +401,8 @@ static int alloc_link(struct pci_dev *dev, int
> > PE_mask, struct ocxl_link **out_l
> > if (rc)
> > goto err_spa;
> >
> > + mutex_init(&link->lpc_mem_lock);
> > +
> > /* platform specific hook */
> > rc = pnv_ocxl_spa_setup(dev, link->spa->spa_mem, PE_mask,
> > &link->platform_data);
> > @@ -711,3 +718,49 @@ void ocxl_link_free_irq(void *link_handle, int
> > hw_irq)
> > atomic_inc(&link->irq_available);
> > }
> > EXPORT_SYMBOL_GPL(ocxl_link_free_irq);
> > +
> > +int ocxl_link_add_lpc_mem(void *link_handle, u64 offset, u64 size)
> > +{
> > + struct ocxl_link *link = (struct ocxl_link *) link_handle;
> > +
> > + // Check for overflow
> > + if (offset > (offset + size))
> > + return -EINVAL;
> > +
> > + mutex_lock(&link->lpc_mem_lock);
> > + link->lpc_mem_sz = max(link->lpc_mem_sz, offset + size);
> > +
> > + mutex_unlock(&link->lpc_mem_lock);
> > +
> > + return 0;
> > +}
> > +
> > +u64 ocxl_link_lpc_map(void *link_handle, struct pci_dev *pdev)
> > +{
> > + struct ocxl_link *link = (struct ocxl_link *) link_handle;
> > +
> > + mutex_lock(&link->lpc_mem_lock);
> > +
> > + if(!link->lpc_mem)
> > + link->lpc_mem = pnv_ocxl_platform_lpc_setup(pdev, link-
> > >lpc_mem_sz);
> > +
> > + if(link->lpc_mem)
> > + link->lpc_consumers++;
> > + mutex_unlock(&link->lpc_mem_lock);
> > +
> > + return link->lpc_mem;
> > +}
> > +
> > +void ocxl_link_lpc_release(void *link_handle, struct pci_dev
> > *pdev)
> > +{
> > + struct ocxl_link *link = (struct ocxl_link *) link_handle;
> > +
> > + mutex_lock(&link->lpc_mem_lock);
> > + WARN_ON(--link->lpc_consumers < 0);
>
> Here, we always decrement the lpc_consumers count. However, it was
> only
> incremented if the mapping was setup correctly in opal.
>
> We could arguably claim that ocxl_link_lpc_release() should only be
> called if ocxl_link_lpc_map() succeeded, but it would make error
> path
> handling easier if we only decrement the lpc_consumers count if
Re: [PATCH v3 03/27] powerpc: Map & release OpenCAPI LPC memory
On Tue, 2020-02-25 at 11:02 +0100, Frederic Barrat wrote:
>
> Le 21/02/2020 à 04:26, Alastair D'Silva a écrit :
> > From: Alastair D'Silva
> >
> > This patch adds platform support to map & release LPC memory.
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > arch/powerpc/include/asm/pnv-ocxl.h | 4 +++
> > arch/powerpc/platforms/powernv/ocxl.c | 43
> > +++
> > 2 files changed, 47 insertions(+)
> >
> > diff --git a/arch/powerpc/include/asm/pnv-ocxl.h
> > b/arch/powerpc/include/asm/pnv-ocxl.h
> > index 7de82647e761..0b2a6707e555 100644
> > --- a/arch/powerpc/include/asm/pnv-ocxl.h
> > +++ b/arch/powerpc/include/asm/pnv-ocxl.h
> > @@ -32,5 +32,9 @@ extern int pnv_ocxl_spa_remove_pe_from_cache(void
> > *platform_data, int pe_handle)
> >
> > extern int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr);
> > extern void pnv_ocxl_free_xive_irq(u32 irq);
> > +#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
> > +u64 pnv_ocxl_platform_lpc_setup(struct pci_dev *pdev, u64 size);
> > +void pnv_ocxl_platform_lpc_release(struct pci_dev *pdev);
> > +#endif
>
> This breaks the compilation of the ocxl driver if
> CONFIG_MEMORY_HOTPLUG=n
>
> Those functions still make sense even without memory hotplug, for
> example in the context of the implementation you had to access
> opencapi
> LPC memory through mmap(). The #ifdef is really needed only around
> the
> check_hotplug_memory_addressable() call.
>
>Fred
Hmm, we do still need sparsemem though. Let me think about his some
more.
>
>
> > #endif /* _ASM_PNV_OCXL_H */
> > diff --git a/arch/powerpc/platforms/powernv/ocxl.c
> > b/arch/powerpc/platforms/powernv/ocxl.c
> > index 8c65aacda9c8..f2edbcc67361 100644
> > --- a/arch/powerpc/platforms/powernv/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/ocxl.c
> > @@ -475,6 +475,49 @@ void pnv_ocxl_spa_release(void *platform_data)
> > }
> > EXPORT_SYMBOL_GPL(pnv_ocxl_spa_release);
> >
> > +#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
> > +u64 pnv_ocxl_platform_lpc_setup(struct pci_dev *pdev, u64 size)
> > +{
> > + struct pci_controller *hose = pci_bus_to_host(pdev->bus);
> > + struct pnv_phb *phb = hose->private_data;
> > + u32 bdfn = pci_dev_id(pdev);
> > + __be64 base_addr_be64;
> > + u64 base_addr;
> > + int rc;
> > +
> > + rc = opal_npu_mem_alloc(phb->opal_id, bdfn, size,
> > &base_addr_be64);
> > + if (rc) {
> > + dev_warn(&pdev->dev,
> > +"OPAL could not allocate LPC memory, rc=%d\n",
> > rc);
> > + return 0;
> > + }
> > +
> > + base_addr = be64_to_cpu(base_addr_be64);
> > +
> > + rc = check_hotplug_memory_addressable(base_addr >> PAGE_SHIFT,
> > + size >> PAGE_SHIFT);
> > + if (rc)
> > + return 0;
> > +
> > + return base_addr;
> > +}
> > +EXPORT_SYMBOL_GPL(pnv_ocxl_platform_lpc_setup);
> > +
> > +void pnv_ocxl_platform_lpc_release(struct pci_dev *pdev)
> > +{
> > + struct pci_controller *hose = pci_bus_to_host(pdev->bus);
> > + struct pnv_phb *phb = hose->private_data;
> > + u32 bdfn = pci_dev_id(pdev);
> > + int rc;
> > +
> > + rc = opal_npu_mem_release(phb->opal_id, bdfn);
> > + if (rc)
> > + dev_warn(&pdev->dev,
> > +"OPAL reported rc=%d when releasing LPC
> > memory\n", rc);
> > +}
> > +EXPORT_SYMBOL_GPL(pnv_ocxl_platform_lpc_release);
> > +#endif
> > +
> > int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int
> > pe_handle)
> > {
> > struct spa_data *data = (struct spa_data *) platform_data;
> >
--
Alastair D'Silva
Open Source Developer
Linux Technology Centre, IBM Australia
mob: 0423 762 819
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
RE: [PATCH v3 00/27] Add support for OpenCAPI Persistent Memory devices
On Mon, 2020-02-24 at 17:51 +1100, Oliver O'Halloran wrote: > On Mon, Feb 24, 2020 at 3:43 PM Alastair D'Silva < > alast...@au1.ibm.com> wrote: > > On Sun, 2020-02-23 at 20:37 -0800, Matthew Wilcox wrote: > > > On Mon, Feb 24, 2020 at 03:34:07PM +1100, Alastair D'Silva wrote: > > > > V3: > > > > - Rebase against next/next-20200220 > > > > - Move driver to arch/powerpc/platforms/powernv, we now > > > > expect > > > > this > > > > driver to go upstream via the powerpc tree > > > > > > That's rather the opposite direction of normal; mostly drivers > > > live > > > under > > > drivers/ and not in arch/. It's easier for drivers to get > > > overlooked > > > when doing tree-wide changes if they're hiding. > > > > This is true, however, given that it was not all that desirable to > > have > > it under drivers/nvdimm, it's sister driver (for the same hardware) > > is > > also under arch, and that we don't expect this driver to be used on > > any > > platform other than powernv, we think this was the most reasonable > > place to put it. > > Historically powernv specific platform drivers go in their respective > subsystem trees rather than in arch/ and I'd prefer we kept it that > way. When I added the papr_scm driver I put it in the pseries > platform > directory because most of the pseries paravirt code lives there for > some reason; I don't know why. Luckily for me that followed the same > model that Dan used when he put the NFIT driver in drivers/acpi/ and > the libnvdimm core in drivers/nvdimm/ so we didn't have anything to > argue about. However, as Matthew pointed out, it is at odds with how > most subsystems operate. Is there any particular reason we're doing > things this way or should we think about moving libnvdimm users to > drivers/nvdimm/? > > Oliver I'm not too fussed where it ends up, as long as it ends up somewhere :) >From what I can tell, the issue is that we have both "infrastructure" drivers, and end-device drivers. To me, it feels like drivers/nvdimm should contain both, and I think this feels like the right approach. I could move it back to drivers/nvdimm/ocxl, but I felt that it was only tolerated there, not desired. This could be cleared up with a response from Dan Williams, and if it is indeed dersired, this is my preferred location. I think a case could also be made for drivers/ocxl, simply because we don't expect more than a handful of drivers to ever live there (I expect most users will drive their devices from userspace via libocxl). In defence of keeping it in arch/powerpc/powernv, I highly doubt this driver will end up being used on any platform other than this. Even though OpenCAPI was engineered as an open standard, there is some competition from industry giants with a competing standard on a much more popular platform. -- Alastair D'Silva Open Source Developer Linux Technology Centre, IBM Australia mob: 0423 762 819 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 08/27] ocxl: Emit a log message showing how much LPC memory was detected
On Mon, 2020-02-24 at 17:06 +1100, Andrew Donnellan wrote: > On 21/2/20 2:27 pm, Alastair D'Silva wrote: > > From: Alastair D'Silva > > > > This patch emits a message showing how much LPC memory & special > > purpose > > memory was detected on an OCXL device. > > > > Signed-off-by: Alastair D'Silva > > --- > > drivers/misc/ocxl/config.c | 4 > > 1 file changed, 4 insertions(+) > > > > diff --git a/drivers/misc/ocxl/config.c > > b/drivers/misc/ocxl/config.c > > index a62e3d7db2bf..701ae6216abf 100644 > > --- a/drivers/misc/ocxl/config.c > > +++ b/drivers/misc/ocxl/config.c > > @@ -568,6 +568,10 @@ static int read_afu_lpc_memory_info(struct > > pci_dev *dev, > > afu->special_purpose_mem_size = > > total_mem_size - lpc_mem_size; > > } > > + > > + dev_info(&dev->dev, "Probed LPC memory of %#llx bytes and > > special purpose memory of %#llx bytes\n", > > + afu->lpc_mem_size, afu->special_purpose_mem_size); > > + > > Printing this at info level for every single AFU seems a bit noisy. > Perhaps we can print it only if LPC memory is > 0? > There is an early exit before this if there is no LPC memory. -- Alastair D'Silva Open Source Developer Linux Technology Centre, IBM Australia mob: 0423 762 819 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 07/27] ocxl: Add functions to map/unmap LPC memory
On Mon, 2020-02-24 at 17:02 +1100, Andrew Donnellan wrote:
> On 21/2/20 2:27 pm, Alastair D'Silva wrote:
> > From: Alastair D'Silva
> >
> > Add functions to map/unmap LPC memory
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > drivers/misc/ocxl/core.c | 51
> > +++
> > drivers/misc/ocxl/ocxl_internal.h | 3 ++
> > include/misc/ocxl.h | 21 +
> > 3 files changed, 75 insertions(+)
> >
> > diff --git a/drivers/misc/ocxl/core.c b/drivers/misc/ocxl/core.c
> > index 2531c6cf19a0..75ff14e3882a 100644
> > --- a/drivers/misc/ocxl/core.c
> > +++ b/drivers/misc/ocxl/core.c
> > @@ -210,6 +210,56 @@ static void unmap_mmio_areas(struct ocxl_afu
> > *afu)
> > release_fn_bar(afu->fn, afu->config.global_mmio_bar);
> > }
> >
> > +int ocxl_afu_map_lpc_mem(struct ocxl_afu *afu)
> > +{
> > + struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
> > +
> > + if ((afu->config.lpc_mem_size + afu-
> > >config.special_purpose_mem_size) == 0)
> > + return 0;
>
> I'd prefer the comparison here to be:
>
>afu->config.lpc_mem_size == 0 &&
> afu->config.special_purpose_mem_size == 0
>
> so a reader doesn't have to think about what this means.
>
Ok
> > +
> > + afu->lpc_base_addr = ocxl_link_lpc_map(afu->fn->link, dev);
> > + if (afu->lpc_base_addr == 0)
> > + return -EINVAL;
> > +
> > + if (afu->config.lpc_mem_size > 0) {
> > + afu->lpc_res.start = afu->lpc_base_addr + afu-
> > >config.lpc_mem_offset;
>
> Maybe not for this series - hmm, I wonder if we should print a
> warning
> somewhere (maybe in read_afu_lpc_memory_info()?) if we see the case
> where (lpc_mem_offset > 0 && lpc_mem_size == 0). Likewise for
> special
> purpose?
>
Sounds reasonable, might as well add it here since there are other LPC
changes.
> > + afu->lpc_res.end = afu->lpc_res.start + afu-
> > >config.lpc_mem_size - 1;
> > + }
> > +
> > + if (afu->config.special_purpose_mem_size > 0) {
> > + afu->special_purpose_res.start = afu->lpc_base_addr +
> > +afu-
> > >config.special_purpose_mem_offset;
> > + afu->special_purpose_res.end = afu-
> > >special_purpose_res.start +
> > + afu-
> > >config.special_purpose_mem_size - 1;
> > + }
> > +
> > + return 0;
> > +}
> > +EXPORT_SYMBOL_GPL(ocxl_afu_map_lpc_mem);
> > +
> > +struct resource *ocxl_afu_lpc_mem(struct ocxl_afu *afu)
> > +{
> > + return &afu->lpc_res;
> > +}
> > +EXPORT_SYMBOL_GPL(ocxl_afu_lpc_mem);
>
> What's the point of this function? A layer of indirection just in
> case
> we need it in future?
>
struct ocxl_afu is opaque outsite the ocxl driver.
> > +
> > +static void unmap_lpc_mem(struct ocxl_afu *afu)
> > +{
> > + struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
> > +
> > + if (afu->lpc_res.start || afu->special_purpose_res.start) {
> > + void *link = afu->fn->link;
> > +
> > + // only release the link when the the last consumer
> > calls release
> > + ocxl_link_lpc_release(link, dev);
> > +
> > + afu->lpc_res.start = 0;
> > + afu->lpc_res.end = 0;
> > + afu->special_purpose_res.start = 0;
> > + afu->special_purpose_res.end = 0;
> > + }
> > +}
> > +
> > static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct
> > pci_dev *dev)
> > {
> > int rc;
> > @@ -251,6 +301,7 @@ static int configure_afu(struct ocxl_afu *afu,
> > u8 afu_idx, struct pci_dev *dev)
> >
> > static void deconfigure_afu(struct ocxl_afu *afu)
> > {
> > + unmap_lpc_mem(afu);
> > unmap_mmio_areas(afu);
> > reclaim_afu_pasid(afu);
> > reclaim_afu_actag(afu);
> > diff --git a/drivers/misc/ocxl/ocxl_internal.h
> > b/drivers/misc/ocxl/ocxl_internal.h
> > index d0c8c4838f42..ce0cac1da416 100644
> > --- a/drivers/misc/ocxl/ocxl_internal.h
> > +++ b/drivers/misc/ocxl/ocxl_internal.h
> > @@ -52,6 +52,9 @@ struct ocxl_afu {
> > void __iomem *global_mmio_ptr;
> > u64 pp_mmio_start;
> > void *private;
> > + u64 lpc_base_addr;
Re: [PATCH v3 01/27] powerpc: Add OPAL calls for LPC memory alloc/release
On Mon, 2020-02-24 at 16:49 +1100, Andrew Donnellan wrote: > On 21/2/20 2:26 pm, Alastair D'Silva wrote: > > From: Alastair D'Silva > > > > Add OPAL calls for LPC memory alloc/release > > > > Signed-off-by: Alastair D'Silva > > Acked-by: Andrew Donnellan > > Acked-by: Frederic Barrat > > Summary line should be "powerpc/powernv". > > Ok -- Alastair D'Silva Open Source Developer Linux Technology Centre, IBM Australia mob: 0423 762 819 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 06/27] ocxl: Tally up the LPC memory on a link & allow it to be mapped
On Mon, 2020-02-24 at 16:25 +1100, Andrew Donnellan wrote: > On 21/2/20 2:26 pm, Alastair D'Silva wrote: > > From: Alastair D'Silva > > > > Tally up the LPC memory on an OpenCAPI link & allow it to be mapped > > > > Signed-off-by: Alastair D'Silva > > This commit message is a bit short and could do with some further > explanation. > > In particular - it's worth explaining why the tracking of available > LPC > memory needs to be done at a link level, because a single OpenCAPI > card > can have multiple PCI functions, each with multiple AFUs which define > an > amount of LPC memory they have, even if the common case is expected > to > be a single function with a single AFU and thus one LPC area per > link. Ok > > Snowpatch has a few checkpatch issues to report: > > https://openpower.xyz/job/snowpatch/job/snowpatch-linux-checkpatch/11800//artifact/linux/checkpatch.log > Gah, I could have sworn I ran checkpatch against this :/ > The code generally looks okay to me. > > > diff --git a/drivers/misc/ocxl/ocxl_internal.h > > b/drivers/misc/ocxl/ocxl_internal.h > > index 198e4e4bc51d..d0c8c4838f42 100644 > > --- a/drivers/misc/ocxl/ocxl_internal.h > > +++ b/drivers/misc/ocxl/ocxl_internal.h > > @@ -142,4 +142,37 @@ int ocxl_irq_offset_to_id(struct ocxl_context > > *ctx, u64 offset); > > u64 ocxl_irq_id_to_offset(struct ocxl_context *ctx, int irq_id); > > void ocxl_afu_irq_free_all(struct ocxl_context *ctx); > > > > +/** > > + * ocxl_link_add_lpc_mem() - Increment the amount of memory > > required by an OpenCAPI link > > + * > > + * @link_handle: The OpenCAPI link handle > > + * @offset: The offset of the memory to add > > + * @size: The amount of memory to increment by > > + * > > + * Returns 0 on success, negative on overflow > > + */ > > I think "amount of memory required" isn't the best way to express > this. > > Might as well explicitly say -EINVAL on overflow. > Ok -- Alastair D'Silva Open Source Developer Linux Technology Centre, IBM Australia mob: 0423 762 819 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 00/27] Add support for OpenCAPI Persistent Memory devices
On Sun, 2020-02-23 at 20:37 -0800, Matthew Wilcox wrote: > On Mon, Feb 24, 2020 at 03:34:07PM +1100, Alastair D'Silva wrote: > > V3: > > - Rebase against next/next-20200220 > > - Move driver to arch/powerpc/platforms/powernv, we now expect > > this > > driver to go upstream via the powerpc tree > > That's rather the opposite direction of normal; mostly drivers live > under > drivers/ and not in arch/. It's easier for drivers to get overlooked > when doing tree-wide changes if they're hiding. This is true, however, given that it was not all that desirable to have it under drivers/nvdimm, it's sister driver (for the same hardware) is also under arch, and that we don't expect this driver to be used on any platform other than powernv, we think this was the most reasonable place to put it. -- Alastair D'Silva Open Source Developer Linux Technology Centre, IBM Australia mob: 0423 762 819 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
Re: [PATCH v3 00/27] Add support for OpenCAPI Persistent Memory devices
On Fri, 2020-02-21 at 08:21 -0800, Dan Williams wrote: > On Thu, Feb 20, 2020 at 7:28 PM Alastair D'Silva < > alast...@au1.ibm.com> wrote: > > From: Alastair D'Silva > > > > This series adds support for OpenCAPI Persistent Memory devices, > > exposing > > them as nvdimms so that we can make use of the existing > > infrastructure. > > A single sentence to introduce: > > 24 files changed, 3029 insertions(+), 97 deletions(-) > > ...is inadequate. What are OpenCAPI Persistent Memory devices? How do > they compare, in terms relevant to libnvdimm, to other persistent > memory devices? What challenges do they pose to the existing > enabling? > What is the overall approach taken with this 27 patch break down? > What > are the changes since v2, v1? If you incorporated someone's review > feedback note it in the cover letter changelog, if you didn't > incorporate someone's feedback note that too with an explanation. > > In short, provide a bridge document for someone familiar with the > upstream infrastructure, but not necessarily steeped in powernv / > OpenCAPI platform details, to get started with this code. > > For now, no need to resend the whole series, just reply to this > message with a fleshed out cover letter and then incorporate it going > forward for v4+. Apologies, I was maintaining a changelog, and forgot to include it. I'll flesh out the cover letter too: This series adds support for OpenCAPI Persistent Memory devices on bare metal (arch/powernv), exposing them as nvdimms so that we can make use of the existing infrastructure. There already exists a driver for the same devices abstracted through PowerVM (arch/pseries): arch/powerpc/platforms/pseries/papr_scm.c These devices are connected via OpenCAPI, and present as LPC (lowest coherence point) memory to the system, practically, that means that memory on these cards could be treated as conventional, cache-coherent memory. Since the devices are connected via OpenCAPI, they are not enumerated via ACPI. Instead, OpenCAPI links present as pseudo-PCI bridges, with devices below them. This series introduces a driver that exposes the memory on these cards as nvdimms, with each card getting it's own bus. This is somewhat complicated by the fact that the cards do not have out of band persistent storage for metadata, so 1 SECTION_SIZE's (see SPARSEMEM) worth of storage is carved out of the top of the card storage to implement the ndctl_config_* calls. The driver is not responsible for configuring the NPU (NVLink Processing Unit) BARs to map the LPC memory from the card into the system's physical address space, instead, it requests this to be done via OPAL calls (typically implemented by Skiboot). The series is structured as follows: - Required infrastructure changes & cleanup - A minimal driver implementation - Implementing additional features within the driver V3: - Rebase against next/next-20200220 - Move driver to arch/powerpc/platforms/powernv, we now expect this driver to go upstream via the powerpc tree - "nvdimm/ocxl: Implement the Read Error Log command" - Fix bad header path - "nvdimm/ocxl: Read the capability registers & wait for device ready" - Fix overlapping masks between readiness_timeout & memory_available_timeout - "nvdimm: Add driver for OpenCAPI Storage Class Memory" - Address minor review comments from Jonathan Cameron - Remove attributes - Default to module if building LIBNVDIMM - Propogate errors up from called functions in probe() - "nvdimm/ocxl: Expose SMART data via ndctl" - Pack attributes in struct - Support different size SMART buffers for compatibility with newer ndctls that may want more SMART attribs than we provide - Rework to to use ND_CMD_CALL instead of ND_CMD_SMART - drop "ocxl: Free detached contexts in ocxl_context_detach_all()" - "powerpc: Map & release OpenCAPI LPC memory" - Remove 'extern' - Only available with CONFIG_MEMORY_HOTPLUG_SPARSE - "ocxl: Tally up the LPC memory on a link & allow it to be mapped" - Address minor review comments from Jonathan Cameron - "ocxl: Add functions to map/unmap LPC memory" - Split detected memory message into a separate patch - Address minor review comments from Jonathan Cameron - Add a comment explaining why unmap_lpc_mem is in deconfigure_afu - "nvdimm/ocxl: Add support for Admin commands" - use sizeof(u64) rather than 0x08 when iterating u64s - "nvdimm/ocxl: Implement the heartbeat command" - Fix typo in blurb - Address kernel doc issues - Ensure all uapi headers use C89 compatible comments - Drop patches
[PATCH v3 12/27] powerpc/powernv/pmem: Add register addresses & status values to the header
From: Alastair D'Silva
These values have been taken from the device specifications.
Signed-off-by: Alastair D'Silva
---
.../platforms/powernv/pmem/ocxl_internal.h| 72 +++
1 file changed, 72 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.h
b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.h
index 0faf3740e9b8..9cf3e42750e7 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.h
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.h
@@ -8,6 +8,78 @@
#define LABEL_AREA_SIZE(1UL << PA_SECTION_SHIFT)
+#define GLOBAL_MMIO_CHI0x000
+#define GLOBAL_MMIO_CHIC 0x008
+#define GLOBAL_MMIO_CHIE 0x010
+#define GLOBAL_MMIO_CHIEC 0x018
+#define GLOBAL_MMIO_HCI0x020
+#define GLOBAL_MMIO_HCIC 0x028
+#define GLOBAL_MMIO_IMA0_OHP 0x040
+#define GLOBAL_MMIO_IMA0_CFP 0x048
+#define GLOBAL_MMIO_IMA1_OHP 0x050
+#define GLOBAL_MMIO_IMA1_CFP 0x058
+#define GLOBAL_MMIO_ACMA_CREQO 0x100
+#define GLOBAL_MMIO_ACMA_CRSPO 0x104
+#define GLOBAL_MMIO_ACMA_CDBO 0x108
+#define GLOBAL_MMIO_ACMA_CDBS 0x10c
+#define GLOBAL_MMIO_NSCMA_CREQO0x120
+#define GLOBAL_MMIO_NSCMA_CRSPO0x124
+#define GLOBAL_MMIO_NSCMA_CDBO 0x128
+#define GLOBAL_MMIO_NSCMA_CDBS 0x12c
+#define GLOBAL_MMIO_CSTS 0x140
+#define GLOBAL_MMIO_FWVER 0x148
+#define GLOBAL_MMIO_CCAP0 0x160
+#define GLOBAL_MMIO_CCAP1 0x168
+
+#define GLOBAL_MMIO_CHI_ACRA BIT_ULL(0)
+#define GLOBAL_MMIO_CHI_NSCRA BIT_ULL(1)
+#define GLOBAL_MMIO_CHI_CRDY BIT_ULL(4)
+#define GLOBAL_MMIO_CHI_CFFS BIT_ULL(5)
+#define GLOBAL_MMIO_CHI_MA BIT_ULL(6)
+#define GLOBAL_MMIO_CHI_ELABIT_ULL(7)
+#define GLOBAL_MMIO_CHI_CDABIT_ULL(8)
+#define GLOBAL_MMIO_CHI_CHFS BIT_ULL(9)
+
+#define GLOBAL_MMIO_CHI_ALL(GLOBAL_MMIO_CHI_ACRA | \
+GLOBAL_MMIO_CHI_NSCRA | \
+GLOBAL_MMIO_CHI_CRDY | \
+GLOBAL_MMIO_CHI_CFFS | \
+GLOBAL_MMIO_CHI_MA | \
+GLOBAL_MMIO_CHI_ELA | \
+GLOBAL_MMIO_CHI_CDA | \
+GLOBAL_MMIO_CHI_CHFS)
+
+#define GLOBAL_MMIO_HCI_ACRW BIT_ULL(0)
+#define GLOBAL_MMIO_HCI_NSCRW BIT_ULL(1)
+#define GLOBAL_MMIO_HCI_AFU_RESET BIT_ULL(2)
+#define GLOBAL_MMIO_HCI_FW_DEBUG BIT_ULL(3)
+#define GLOBAL_MMIO_HCI_CONTROLLER_DUMPBIT_ULL(4)
+#define GLOBAL_MMIO_HCI_CONTROLLER_DUMP_COLLECTED BIT_ULL(5)
+#define GLOBAL_MMIO_HCI_REQ_HEALTH_PERFBIT_ULL(6)
+
+#define ADMIN_COMMAND_HEARTBEAT0x00u
+#define ADMIN_COMMAND_SHUTDOWN 0x01u
+#define ADMIN_COMMAND_FW_UPDATE0x02u
+#define ADMIN_COMMAND_FW_DEBUG 0x03u
+#define ADMIN_COMMAND_ERRLOG 0x04u
+#define ADMIN_COMMAND_SMART0x05u
+#define ADMIN_COMMAND_CONTROLLER_STATS 0x06u
+#define ADMIN_COMMAND_CONTROLLER_DUMP 0x07u
+#define ADMIN_COMMAND_CMD_CAPS 0x08u
+#define ADMIN_COMMAND_MAX 0x08u
+
+#define STATUS_SUCCESS 0x00
+#define STATUS_MEM_UNAVAILABLE 0x20
+#define STATUS_BAD_OPCODE 0x50
+#define STATUS_BAD_REQUEST_PARM0x51
+#define STATUS_BAD_DATA_PARM 0x52
+#define STATUS_DEBUG_BLOCKED 0x70
+#define STATUS_FAIL0xFF
+
+#define STATUS_FW_UPDATE_BLOCKED 0x21
+#define STATUS_FW_ARG_INVALID 0x51
+#define STATUS_FW_INVALID 0x52
+
struct ocxlpmem_function0 {
struct pci_dev *pdev;
struct ocxl_fn *ocxl_fn;
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 21/27] powerpc/powernv/pmem: Add an IOCTL to request controller health & perf data
From: Alastair D'Silva
When health & performance data is requested from the controller,
it responds with an error log containing the requested information.
This patch allows the request to me issued via an IOCTL.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/ocxl.c | 16
include/uapi/nvdimm/ocxl-pmem.h| 1 +
2 files changed, 17 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index e46696d3cc36..081883a8247a 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -1000,6 +1000,18 @@ static int ioctl_event_check(struct ocxlpmem *ocxlpmem,
u64 __user *uarg)
return rc;
}
+/**
+ * req_controller_health_perf() - Request controller health & performance data
+ * @ocxlpmem: the device metadata
+ * Return: 0 on success, negative on failure
+ */
+int req_controller_health_perf(struct ocxlpmem *ocxlpmem)
+{
+ return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_HCI,
+ OCXL_LITTLE_ENDIAN,
+ GLOBAL_MMIO_HCI_REQ_HEALTH_PERF);
+}
+
static long file_ioctl(struct file *file, unsigned int cmd, unsigned long args)
{
struct ocxlpmem *ocxlpmem = file->private_data;
@@ -1037,6 +1049,10 @@ static long file_ioctl(struct file *file, unsigned int
cmd, unsigned long args)
case IOCTL_OCXL_PMEM_EVENT_CHECK:
rc = ioctl_event_check(ocxlpmem, (u64 __user *)args);
break;
+
+ case IOCTL_OCXL_PMEM_REQUEST_HEALTH:
+ rc = req_controller_health_perf(ocxlpmem);
+ break;
}
return rc;
diff --git a/include/uapi/nvdimm/ocxl-pmem.h b/include/uapi/nvdimm/ocxl-pmem.h
index 988eb0bc413d..0d03abb44001 100644
--- a/include/uapi/nvdimm/ocxl-pmem.h
+++ b/include/uapi/nvdimm/ocxl-pmem.h
@@ -90,5 +90,6 @@ struct ioctl_ocxl_pmem_eventfd {
#define IOCTL_OCXL_PMEM_CONTROLLER_STATS _IO(OCXL_PMEM_MAGIC,
0x05)
#define IOCTL_OCXL_PMEM_EVENTFD
_IOW(OCXL_PMEM_MAGIC, 0x06, struct ioctl_ocxl_pmem_eventfd)
#define IOCTL_OCXL_PMEM_EVENT_CHECK_IOR(OCXL_PMEM_MAGIC,
0x07, __u64)
+#define IOCTL_OCXL_PMEM_REQUEST_HEALTH _IO(OCXL_PMEM_MAGIC,
0x08)
#endif /* _UAPI_OCXL_SCM_H */
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 16/27] powerpc/powernv/pmem: Register a character device for userspace to interact with
From: Alastair D'Silva
This patch introduces a character device (/dev/ocxl-scmX) which further
patches will use to interact with userspace.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/ocxl.c| 116 +-
.../platforms/powernv/pmem/ocxl_internal.h| 2 +
2 files changed, 116 insertions(+), 2 deletions(-)
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index b8bd7e703b19..63109a870d2c 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -10,6 +10,7 @@
#include
#include
#include
+#include
#include
#include
#include "ocxl_internal.h"
@@ -339,6 +340,9 @@ static void free_ocxlpmem(struct ocxlpmem *ocxlpmem)
free_minor(ocxlpmem);
+ if (ocxlpmem->cdev.owner)
+ cdev_del(&ocxlpmem->cdev);
+
if (ocxlpmem->metadata_addr)
devm_memunmap(&ocxlpmem->dev, ocxlpmem->metadata_addr);
@@ -396,6 +400,70 @@ static int ocxlpmem_register(struct ocxlpmem *ocxlpmem)
return device_register(&ocxlpmem->dev);
}
+static void ocxlpmem_put(struct ocxlpmem *ocxlpmem)
+{
+ put_device(&ocxlpmem->dev);
+}
+
+static struct ocxlpmem *ocxlpmem_get(struct ocxlpmem *ocxlpmem)
+{
+ return (get_device(&ocxlpmem->dev) == NULL) ? NULL : ocxlpmem;
+}
+
+static struct ocxlpmem *find_and_get_ocxlpmem(dev_t devno)
+{
+ struct ocxlpmem *ocxlpmem;
+ int minor = MINOR(devno);
+ /*
+* We don't declare an RCU critical section here, as our AFU
+* is protected by a reference counter on the device. By the time the
+* minor number of a device is removed from the idr, the ref count of
+* the device is already at 0, so no user API will access that AFU and
+* this function can't return it.
+*/
+ ocxlpmem = idr_find(&minors_idr, minor);
+ if (ocxlpmem)
+ ocxlpmem_get(ocxlpmem);
+ return ocxlpmem;
+}
+
+static int file_open(struct inode *inode, struct file *file)
+{
+ struct ocxlpmem *ocxlpmem;
+
+ ocxlpmem = find_and_get_ocxlpmem(inode->i_rdev);
+ if (!ocxlpmem)
+ return -ENODEV;
+
+ file->private_data = ocxlpmem;
+ return 0;
+}
+
+static int file_release(struct inode *inode, struct file *file)
+{
+ struct ocxlpmem *ocxlpmem = file->private_data;
+
+ ocxlpmem_put(ocxlpmem);
+ return 0;
+}
+
+static const struct file_operations fops = {
+ .owner = THIS_MODULE,
+ .open = file_open,
+ .release= file_release,
+};
+
+/**
+ * create_cdev() - Create the chardev in /dev for the device
+ * @ocxlpmem: the SCM metadata
+ * Return: 0 on success, negative on failure
+ */
+static int create_cdev(struct ocxlpmem *ocxlpmem)
+{
+ cdev_init(&ocxlpmem->cdev, &fops);
+ return cdev_add(&ocxlpmem->cdev, ocxlpmem->dev.devt, 1);
+}
+
/**
* ocxlpmem_remove() - Free an OpenCAPI persistent memory device
* @pdev: the PCI device information struct
@@ -572,6 +640,11 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
goto err;
}
+ if (create_cdev(ocxlpmem)) {
+ dev_err(&pdev->dev, "Could not create character device\n");
+ goto err;
+ }
+
elapsed = 0;
timeout = ocxlpmem->readiness_timeout +
ocxlpmem->memory_available_timeout;
while (!is_usable(ocxlpmem, false)) {
@@ -613,20 +686,59 @@ static struct pci_driver pci_driver = {
.shutdown = ocxlpmem_remove,
};
+static int file_init(void)
+{
+ int rc;
+
+ mutex_init(&minors_idr_lock);
+ idr_init(&minors_idr);
+
+ rc = alloc_chrdev_region(&ocxlpmem_dev, 0, NUM_MINORS, "ocxl-pmem");
+ if (rc) {
+ idr_destroy(&minors_idr);
+ pr_err("Unable to allocate OpenCAPI persistent memory major
number: %d\n", rc);
+ return rc;
+ }
+
+ ocxlpmem_class = class_create(THIS_MODULE, "ocxl-pmem");
+ if (IS_ERR(ocxlpmem_class)) {
+ idr_destroy(&minors_idr);
+ pr_err("Unable to create ocxl-pmem class\n");
+ unregister_chrdev_region(ocxlpmem_dev, NUM_MINORS);
+ return PTR_ERR(ocxlpmem_class);
+ }
+
+ return 0;
+}
+
+static void file_exit(void)
+{
+ class_destroy(ocxlpmem_class);
+ unregister_chrdev_region(ocxlpmem_dev, NUM_MINORS);
+ idr_destroy(&minors_idr);
+}
+
static int __init ocxlpmem_init(void)
{
- int rc = 0;
+ int rc;
- rc = pci_register_driver(&pci_driver);
+ rc = file_init();
if (rc)
return rc;
+ rc = pci_register_driver(&pci_driver);
+
[PATCH v3 24/27] powerpc/powernv/pmem: Expose SMART data via ndctl
From: Alastair D'Silva
This patch retrieves proprietary formatted SMART data and makes it
available via ndctl. A later contribution will be made to ndctl to
parse this data.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/ocxl.c| 128 ++
.../platforms/powernv/pmem/ocxl_internal.h| 18 +++
include/uapi/linux/ndctl.h| 1 +
3 files changed, 147 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index d4ce5e9e0521..5cd1b6d78dd6 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -81,6 +81,129 @@ static int ndctl_config_size(struct nd_cmd_get_config_size
*command)
return 0;
}
+/**
+ * smart_header_parse() - Parse the first 64 bits of the SMART admin command
response
+ * @ocxlpmem: the device metadata
+ * @length: out, returns the number of bytes in the response (excluding the 64
bit header)
+ */
+static int smart_header_parse(struct ocxlpmem *ocxlpmem, u32 *length)
+{
+ int rc;
+ u64 val;
+
+ u16 data_identifier;
+ u32 data_length;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ return rc;
+
+ data_identifier = val >> 48;
+ data_length = val & 0x;
+
+ if (data_identifier != 0x534D) { // 'SM'
+ dev_err(&ocxlpmem->dev,
+ "Bad data identifier for smart data, expected 'SM', got
'%-.*s'\n",
+ 2, (char *)&data_identifier);
+ return -EINVAL;
+ }
+
+ *length = data_length;
+ return 0;
+}
+
+static int ndctl_smart(struct ocxlpmem *ocxlpmem, struct nd_cmd_pkg *pkg)
+{
+ u32 length, i;
+ struct nd_ocxl_smart *out;
+ int rc;
+
+ mutex_lock(&ocxlpmem->admin_command.lock);
+
+ rc = admin_command_request(ocxlpmem, ADMIN_COMMAND_SMART);
+ if (rc)
+ goto out;
+
+ rc = admin_command_execute(ocxlpmem);
+ if (rc)
+ goto out;
+
+ rc = admin_command_complete_timeout(ocxlpmem, ADMIN_COMMAND_SMART);
+ if (rc < 0) {
+ dev_err(&ocxlpmem->dev, "SMART timeout\n");
+ goto out;
+ }
+
+ rc = admin_response(ocxlpmem);
+ if (rc < 0)
+ goto out;
+ if (rc != STATUS_SUCCESS) {
+ warn_status(ocxlpmem, "Unexpected status from SMART", rc);
+ goto out;
+ }
+
+ rc = smart_header_parse(ocxlpmem, &length);
+ if (rc)
+ goto out;
+
+ pkg->nd_fw_size = length;
+
+ length = min(length, pkg->nd_size_out); // bytes
+ out = (struct nd_ocxl_smart *)pkg->nd_payload;
+ // Each SMART attribute is 2 * 64 bits
+ out->count = length / (2 * sizeof(u64)); // attributes
+
+ for (i = 0; i < length; i += sizeof(u64)) {
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+
ocxlpmem->admin_command.data_offset + sizeof(u64) + i,
+OCXL_LITTLE_ENDIAN,
+&out->attribs[i/sizeof(u64)]);
+ if (rc)
+ goto out;
+ }
+
+ rc = admin_response_handled(ocxlpmem);
+ if (rc)
+ goto out;
+
+ rc = 0;
+ goto out;
+
+out:
+ mutex_unlock(&ocxlpmem->admin_command.lock);
+ return rc;
+}
+
+static int ndctl_call(struct ocxlpmem *ocxlpmem, void *buf, unsigned int
buf_len)
+{
+ struct nd_cmd_pkg *pkg = buf;
+
+ if (buf_len < sizeof(struct nd_cmd_pkg)) {
+ dev_err(&ocxlpmem->dev, "Invalid ND_CALL size=%u\n", buf_len);
+ return -EINVAL;
+ }
+
+ if (pkg->nd_family != NVDIMM_FAMILY_OCXL) {
+ dev_err(&ocxlpmem->dev, "Invalid ND_CALL family=0x%llx\n",
pkg->nd_family);
+ return -EINVAL;
+ }
+
+ switch (pkg->nd_command) {
+ case ND_CMD_OCXL_SMART:
+ ndctl_smart(ocxlpmem, pkg);
+ break;
+
+ default:
+ dev_err(&ocxlpmem->dev, "Invalid ND_CALL command=0x%llx\n",
pkg->nd_command);
+ return -EINVAL;
+ }
+
+
+ return 0;
+}
+
static int ndctl(struct nvdimm_bus_descriptor *nd_desc,
struct nvdimm *nvdimm,
unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc)
@@ -88,6 +211,10 @@ static int ndctl(struct nvdimm_bus_descriptor *nd_desc,
struct ocxlpmem *ocxlpmem = c
[PATCH v3 19/27] powerpc/powernv/pmem: Add an IOCTL to report controller statistics
From: Alastair D'Silva
The controller can report a number of statistics that are useful
in evaluating the performance and reliability of the card.
This patch exposes this information via an IOCTL.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/ocxl.c | 185 +
include/uapi/nvdimm/ocxl-pmem.h| 17 ++
2 files changed, 202 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index 2cabafe1fc58..009d4fd29e7d 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -758,6 +758,186 @@ static int ioctl_controller_dump_complete(struct ocxlpmem
*ocxlpmem)
GLOBAL_MMIO_HCI_CONTROLLER_DUMP_COLLECTED);
}
+/**
+ * controller_stats_header_parse() - Parse the first 64 bits of the controller
stats admin command response
+ * @ocxlpmem: the device metadata
+ * @length: out, returns the number of bytes in the response (excluding the 64
bit header)
+ */
+static int controller_stats_header_parse(struct ocxlpmem *ocxlpmem,
+ u32 *length)
+{
+ int rc;
+ u64 val;
+
+ u16 data_identifier;
+ u32 data_length;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ return rc;
+
+ data_identifier = val >> 48;
+ data_length = val & 0x;
+
+ if (data_identifier != 0x4353) { // 'CS'
+ dev_err(&ocxlpmem->dev,
+ "Bad data identifier for controller stats, expected
'CS', got '%-.*s'\n",
+ 2, (char *)&data_identifier);
+ return -EINVAL;
+ }
+
+ *length = data_length;
+ return 0;
+}
+
+static int ioctl_controller_stats(struct ocxlpmem *ocxlpmem,
+ struct ioctl_ocxl_pmem_controller_stats
__user *uarg)
+{
+ struct ioctl_ocxl_pmem_controller_stats args;
+ u32 length;
+ int rc;
+ u64 val;
+
+ memset(&args, '\0', sizeof(args));
+
+ mutex_lock(&ocxlpmem->admin_command.lock);
+
+ rc = admin_command_request(ocxlpmem, ADMIN_COMMAND_CONTROLLER_STATS);
+ if (rc)
+ goto out;
+
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+ ocxlpmem->admin_command.request_offset +
0x08,
+ OCXL_LITTLE_ENDIAN, 0);
+ if (rc)
+ goto out;
+
+ rc = admin_command_execute(ocxlpmem);
+ if (rc)
+ goto out;
+
+
+ rc = admin_command_complete_timeout(ocxlpmem,
+ ADMIN_COMMAND_CONTROLLER_STATS);
+ if (rc < 0) {
+ dev_warn(&ocxlpmem->dev, "Controller stats timed out\n");
+ goto out;
+ }
+
+ rc = admin_response(ocxlpmem);
+ if (rc < 0)
+ goto out;
+ if (rc != STATUS_SUCCESS) {
+ warn_status(ocxlpmem,
+ "Unexpected status from controller stats", rc);
+ goto out;
+ }
+
+ rc = controller_stats_header_parse(ocxlpmem, &length);
+ if (rc)
+ goto out;
+
+ if (length != 0x140)
+ warn_status(ocxlpmem,
+ "Unexpected length for controller stats data,
expected 0x140, got 0x%x",
+ length);
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset + 0x08
+ 0x08,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ goto out;
+
+ args.reset_count = val >> 32;
+ args.reset_uptime = val & 0x;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset + 0x08
+ 0x10,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ goto out;
+
+ args.power_on_uptime = val >> 32;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset + 0x08
+ 0x40 + 0x08,
+OCXL_LITTLE_ENDIAN, &args.host_load_count);
+ if (rc)
+ goto out;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset + 0x08
+ 0x40 + 0x10,
+OCXL_LITTLE_ENDIAN,
&args.host_store_count);
+ if (rc)
+ goto out;
[PATCH v3 23/27] powerpc/powernv/pmem: Add debug IOCTLs
From: Alastair D'Silva
These IOCTLs provide low level access to the card to aid in debugging
controller/FPGA firmware.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/Kconfig | 6 +
arch/powerpc/platforms/powernv/pmem/ocxl.c | 249
include/uapi/nvdimm/ocxl-pmem.h | 32 +++
3 files changed, 287 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pmem/Kconfig
b/arch/powerpc/platforms/powernv/pmem/Kconfig
index c5d927520920..3f44429d70c9 100644
--- a/arch/powerpc/platforms/powernv/pmem/Kconfig
+++ b/arch/powerpc/platforms/powernv/pmem/Kconfig
@@ -12,4 +12,10 @@ config OCXL_PMEM
Select N if unsure.
+config OCXL_PMEM_DEBUG
+ bool "OpenCAPI Persistent Memory debugging"
+ depends on OCXL_PMEM
+ help
+ Enables low level IOCTLs for OpenCAPI Persistent Memory firmware
development
+
endif
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index e01f6f9fc180..d4ce5e9e0521 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -1050,6 +1050,235 @@ int req_controller_health_perf(struct ocxlpmem
*ocxlpmem)
GLOBAL_MMIO_HCI_REQ_HEALTH_PERF);
}
+#ifdef CONFIG_OCXL_PMEM_DEBUG
+/**
+ * enable_fwdebug() - Enable FW debug on the controller
+ * @ocxlpmem: the device metadata
+ * Return: 0 on success, negative on failure
+ */
+static int enable_fwdebug(const struct ocxlpmem *ocxlpmem)
+{
+ return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_HCI,
+ OCXL_LITTLE_ENDIAN,
+ GLOBAL_MMIO_HCI_FW_DEBUG);
+}
+
+/**
+ * disable_fwdebug() - Disable FW debug on the controller
+ * @ocxlpmem: the device metadata
+ * Return: 0 on success, negative on failure
+ */
+static int disable_fwdebug(const struct ocxlpmem *ocxlpmem)
+{
+ return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_HCIC,
+ OCXL_LITTLE_ENDIAN,
+ GLOBAL_MMIO_HCI_FW_DEBUG);
+}
+
+static int ioctl_fwdebug(struct ocxlpmem *ocxlpmem,
+struct ioctl_ocxl_pmem_fwdebug __user *uarg)
+{
+ struct ioctl_ocxl_pmem_fwdebug args;
+ u64 val;
+ int i;
+ int rc;
+
+ if (copy_from_user(&args, uarg, sizeof(args)))
+ return -EFAULT;
+
+ // Buffer size must be a multiple of 8
+ if ((args.buf_size & 0x07))
+ return -EINVAL;
+
+ if (args.buf_size > ocxlpmem->admin_command.data_size)
+ return -EINVAL;
+
+ mutex_lock(&ocxlpmem->admin_command.lock);
+
+ rc = enable_fwdebug(ocxlpmem);
+ if (rc)
+ goto out;
+
+ rc = admin_command_request(ocxlpmem, ADMIN_COMMAND_FW_DEBUG);
+ if (rc)
+ goto out;
+
+ // Write DebugAction & FunctionCode
+ val = ((u64)args.debug_action << 56) | ((u64)args.function_code << 40);
+
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+ ocxlpmem->admin_command.request_offset +
0x08,
+ OCXL_LITTLE_ENDIAN, val);
+ if (rc)
+ goto out;
+
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+ ocxlpmem->admin_command.request_offset +
0x10,
+ OCXL_LITTLE_ENDIAN,
args.debug_parameter_1);
+ if (rc)
+ goto out;
+
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+ ocxlpmem->admin_command.request_offset +
0x18,
+ OCXL_LITTLE_ENDIAN,
args.debug_parameter_2);
+ if (rc)
+ goto out;
+
+ for (i = 0x20; i < 0x38; i += 0x08)
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+
ocxlpmem->admin_command.request_offset + i,
+ OCXL_LITTLE_ENDIAN, 0);
+ if (rc)
+ goto out;
+
+
+ // Populate admin command buffer
+ if (args.buf_size) {
+ for (i = 0; i < args.buf_size; i += sizeof(u64)) {
+ u64 val;
+
+ if (copy_from_user(&val, &args.buf[i], sizeof(u64)))
+ return -EFAULT;
+
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+
ocxlpmem->admin_command.data_offset + i,
+ OCXL_HOST_ENDIAN, val);
+ if (rc)
+ goto out;
+ }
+ }
+
+ rc = admin_command_execute(
[PATCH v3 17/27] powerpc/powernv/pmem: Implement the Read Error Log command
From: Alastair D'Silva
The read error log command extracts information from the controller's
internal error log.
This patch exposes this information in 2 ways:
- During probe, if an error occurs & a log is available, print it to the
console
- After probe, make the error log available to userspace via an IOCTL.
Userspace is notified of pending error logs in a later patch
("powerpc/powernv/pmem: Forward events to userspace")
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/ocxl.c| 269 ++
.../platforms/powernv/pmem/ocxl_internal.h| 1 +
include/uapi/nvdimm/ocxl-pmem.h | 46 +++
3 files changed, 316 insertions(+)
create mode 100644 include/uapi/nvdimm/ocxl-pmem.h
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index 63109a870d2c..2b64504f9129 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -447,10 +447,219 @@ static int file_release(struct inode *inode, struct file
*file)
return 0;
}
+/**
+ * error_log_header_parse() - Parse the first 64 bits of the error log command
response
+ * @ocxlpmem: the device metadata
+ * @length: out, returns the number of bytes in the response (excluding the 64
bit header)
+ */
+static int error_log_header_parse(struct ocxlpmem *ocxlpmem, u16 *length)
+{
+ int rc;
+ u64 val;
+
+ u16 data_identifier;
+ u32 data_length;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ return rc;
+
+ data_identifier = val >> 48;
+ data_length = val & 0x;
+
+ if (data_identifier != 0x454C) { // 'EL'
+ dev_err(&ocxlpmem->dev,
+ "Bad data identifier for error log data, expected 'EL',
got '%2s' (%#x), data_length=%u\n",
+ (char *)&data_identifier,
+ (unsigned int)data_identifier, data_length);
+ return -EINVAL;
+ }
+
+ *length = data_length;
+ return 0;
+}
+
+static int error_log_offset_0x08(struct ocxlpmem *ocxlpmem,
+u32 *log_identifier, u32 *program_ref_code)
+{
+ int rc;
+ u64 val;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset + 0x08,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ return rc;
+
+ *log_identifier = val >> 32;
+ *program_ref_code = val & 0x;
+
+ return 0;
+}
+
+static int read_error_log(struct ocxlpmem *ocxlpmem,
+ struct ioctl_ocxl_pmem_error_log *log, bool
buf_is_user)
+{
+ u64 val;
+ u16 user_buf_length;
+ u16 buf_length;
+ u16 i;
+ int rc;
+
+ if (log->buf_size % 8)
+ return -EINVAL;
+
+ rc = ocxlpmem_chi(ocxlpmem, &val);
+ if (rc)
+ goto out;
+
+ if (!(val & GLOBAL_MMIO_CHI_ELA))
+ return -EAGAIN;
+
+ user_buf_length = log->buf_size;
+
+ mutex_lock(&ocxlpmem->admin_command.lock);
+
+ rc = admin_command_request(ocxlpmem, ADMIN_COMMAND_ERRLOG);
+ if (rc)
+ goto out;
+
+ rc = admin_command_execute(ocxlpmem);
+ if (rc)
+ goto out;
+
+ rc = admin_command_complete_timeout(ocxlpmem, ADMIN_COMMAND_ERRLOG);
+ if (rc < 0) {
+ dev_warn(&ocxlpmem->dev, "Read error log timed out\n");
+ goto out;
+ }
+
+ rc = admin_response(ocxlpmem);
+ if (rc < 0)
+ goto out;
+ if (rc != STATUS_SUCCESS) {
+ warn_status(ocxlpmem, "Unexpected status from retrieve error
log", rc);
+ goto out;
+ }
+
+
+ rc = error_log_header_parse(ocxlpmem, &log->buf_size);
+ if (rc)
+ goto out;
+ // log->buf_size now contains the returned buffer size, not the user
size
+
+ rc = error_log_offset_0x08(ocxlpmem, &log->log_identifier,
+ &log->program_reference_code);
+ if (rc)
+ goto out;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+ocxlpmem->admin_command.data_offset + 0x10,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ goto out;
+
+ log->error_log_type = val >> 56;
+ log->action_flags = (log->error_log_type ==
OCXL_PMEM_ERROR_LOG_TYPE_GENERAL) ?
+
[PATCH v3 01/27] powerpc: Add OPAL calls for LPC memory alloc/release
From: Alastair D'Silva Add OPAL calls for LPC memory alloc/release Signed-off-by: Alastair D'Silva Acked-by: Andrew Donnellan Acked-by: Frederic Barrat --- arch/powerpc/include/asm/opal-api.h| 2 ++ arch/powerpc/include/asm/opal.h| 3 +++ arch/powerpc/platforms/powernv/opal-call.c | 2 ++ 3 files changed, 7 insertions(+) diff --git a/arch/powerpc/include/asm/opal-api.h b/arch/powerpc/include/asm/opal-api.h index c1f25a760eb1..9298e603001b 100644 --- a/arch/powerpc/include/asm/opal-api.h +++ b/arch/powerpc/include/asm/opal-api.h @@ -208,6 +208,8 @@ #define OPAL_HANDLE_HMI2 166 #defineOPAL_NX_COPROC_INIT 167 #define OPAL_XIVE_GET_VP_STATE 170 +#define OPAL_NPU_MEM_ALLOC 171 +#define OPAL_NPU_MEM_RELEASE 172 #define OPAL_MPIPL_UPDATE 173 #define OPAL_MPIPL_REGISTER_TAG174 #define OPAL_MPIPL_QUERY_TAG 175 diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h index 9986ac34b8e2..8f7727e0f9ce 100644 --- a/arch/powerpc/include/asm/opal.h +++ b/arch/powerpc/include/asm/opal.h @@ -39,6 +39,9 @@ int64_t opal_npu_spa_clear_cache(uint64_t phb_id, uint32_t bdfn, uint64_t PE_handle); int64_t opal_npu_tl_set(uint64_t phb_id, uint32_t bdfn, long cap, uint64_t rate_phys, uint32_t size); +int64_t opal_npu_mem_alloc(uint64_t phb_id, uint32_t bdfn, + uint64_t size, uint64_t *bar); +int64_t opal_npu_mem_release(uint64_t phb_id, uint32_t bdfn); int64_t opal_console_write(int64_t term_number, __be64 *length, const uint8_t *buffer); diff --git a/arch/powerpc/platforms/powernv/opal-call.c b/arch/powerpc/platforms/powernv/opal-call.c index 5cd0f52d258f..f26e58b72c04 100644 --- a/arch/powerpc/platforms/powernv/opal-call.c +++ b/arch/powerpc/platforms/powernv/opal-call.c @@ -287,6 +287,8 @@ OPAL_CALL(opal_pci_set_pbcq_tunnel_bar, OPAL_PCI_SET_PBCQ_TUNNEL_BAR); OPAL_CALL(opal_sensor_read_u64,OPAL_SENSOR_READ_U64); OPAL_CALL(opal_sensor_group_enable,OPAL_SENSOR_GROUP_ENABLE); OPAL_CALL(opal_nx_coproc_init, OPAL_NX_COPROC_INIT); +OPAL_CALL(opal_npu_mem_alloc, OPAL_NPU_MEM_ALLOC); +OPAL_CALL(opal_npu_mem_release,OPAL_NPU_MEM_RELEASE); OPAL_CALL(opal_mpipl_update, OPAL_MPIPL_UPDATE); OPAL_CALL(opal_mpipl_register_tag, OPAL_MPIPL_REGISTER_TAG); OPAL_CALL(opal_mpipl_query_tag,OPAL_MPIPL_QUERY_TAG); -- 2.24.1 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 04/27] ocxl: Remove unnecessary externs
From: Alastair D'Silva Function declarations don't need externs, remove the existing ones so they are consistent with newer code Signed-off-by: Alastair D'Silva --- arch/powerpc/include/asm/pnv-ocxl.h | 32 ++--- include/misc/ocxl.h | 6 +++--- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/arch/powerpc/include/asm/pnv-ocxl.h b/arch/powerpc/include/asm/pnv-ocxl.h index 0b2a6707e555..b23c99bc0c84 100644 --- a/arch/powerpc/include/asm/pnv-ocxl.h +++ b/arch/powerpc/include/asm/pnv-ocxl.h @@ -9,29 +9,27 @@ #define PNV_OCXL_TL_BITS_PER_RATE 4 #define PNV_OCXL_TL_RATE_BUF_SIZE ((PNV_OCXL_TL_MAX_TEMPLATE+1) * PNV_OCXL_TL_BITS_PER_RATE / 8) -extern int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled, - u16 *supported); -extern int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count); +int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled, u16 *supported); +int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count); -extern int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap, +int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap, char *rate_buf, int rate_buf_size); -extern int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap, +int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap, uint64_t rate_buf_phys, int rate_buf_size); -extern int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq); -extern void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar, - void __iomem *tfc, void __iomem *pe_handle); -extern int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr, - void __iomem **dar, void __iomem **tfc, - void __iomem **pe_handle); +int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq); +void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar, +void __iomem *tfc, void __iomem *pe_handle); +int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr, + void __iomem **dar, void __iomem **tfc, + void __iomem **pe_handle); -extern int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask, - void **platform_data); -extern void pnv_ocxl_spa_release(void *platform_data); -extern int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle); +int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask, void **platform_data); +void pnv_ocxl_spa_release(void *platform_data); +int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle); -extern int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr); -extern void pnv_ocxl_free_xive_irq(u32 irq); +int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr); +void pnv_ocxl_free_xive_irq(u32 irq); #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE u64 pnv_ocxl_platform_lpc_setup(struct pci_dev *pdev, u64 size); void pnv_ocxl_platform_lpc_release(struct pci_dev *pdev); diff --git a/include/misc/ocxl.h b/include/misc/ocxl.h index 06dd5839e438..0a762e387418 100644 --- a/include/misc/ocxl.h +++ b/include/misc/ocxl.h @@ -173,7 +173,7 @@ int ocxl_context_detach(struct ocxl_context *ctx); * * Returns 0 on success, negative on failure */ -extern int ocxl_afu_irq_alloc(struct ocxl_context *ctx, int *irq_id); +int ocxl_afu_irq_alloc(struct ocxl_context *ctx, int *irq_id); /** * Frees an IRQ associated with an AFU context @@ -182,7 +182,7 @@ extern int ocxl_afu_irq_alloc(struct ocxl_context *ctx, int *irq_id); * * Returns 0 on success, negative on failure */ -extern int ocxl_afu_irq_free(struct ocxl_context *ctx, int irq_id); +int ocxl_afu_irq_free(struct ocxl_context *ctx, int irq_id); /** * Gets the address of the trigger page for an IRQ @@ -193,7 +193,7 @@ extern int ocxl_afu_irq_free(struct ocxl_context *ctx, int irq_id); * * returns the trigger page address, or 0 if the IRQ is not valid */ -extern u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, int irq_id); +u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, int irq_id); /** * Provide a callback to be called when an IRQ is triggered -- 2.24.1 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 03/27] powerpc: Map & release OpenCAPI LPC memory
From: Alastair D'Silva
This patch adds platform support to map & release LPC memory.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/include/asm/pnv-ocxl.h | 4 +++
arch/powerpc/platforms/powernv/ocxl.c | 43 +++
2 files changed, 47 insertions(+)
diff --git a/arch/powerpc/include/asm/pnv-ocxl.h
b/arch/powerpc/include/asm/pnv-ocxl.h
index 7de82647e761..0b2a6707e555 100644
--- a/arch/powerpc/include/asm/pnv-ocxl.h
+++ b/arch/powerpc/include/asm/pnv-ocxl.h
@@ -32,5 +32,9 @@ extern int pnv_ocxl_spa_remove_pe_from_cache(void
*platform_data, int pe_handle)
extern int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr);
extern void pnv_ocxl_free_xive_irq(u32 irq);
+#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
+u64 pnv_ocxl_platform_lpc_setup(struct pci_dev *pdev, u64 size);
+void pnv_ocxl_platform_lpc_release(struct pci_dev *pdev);
+#endif
#endif /* _ASM_PNV_OCXL_H */
diff --git a/arch/powerpc/platforms/powernv/ocxl.c
b/arch/powerpc/platforms/powernv/ocxl.c
index 8c65aacda9c8..f2edbcc67361 100644
--- a/arch/powerpc/platforms/powernv/ocxl.c
+++ b/arch/powerpc/platforms/powernv/ocxl.c
@@ -475,6 +475,49 @@ void pnv_ocxl_spa_release(void *platform_data)
}
EXPORT_SYMBOL_GPL(pnv_ocxl_spa_release);
+#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
+u64 pnv_ocxl_platform_lpc_setup(struct pci_dev *pdev, u64 size)
+{
+ struct pci_controller *hose = pci_bus_to_host(pdev->bus);
+ struct pnv_phb *phb = hose->private_data;
+ u32 bdfn = pci_dev_id(pdev);
+ __be64 base_addr_be64;
+ u64 base_addr;
+ int rc;
+
+ rc = opal_npu_mem_alloc(phb->opal_id, bdfn, size, &base_addr_be64);
+ if (rc) {
+ dev_warn(&pdev->dev,
+"OPAL could not allocate LPC memory, rc=%d\n", rc);
+ return 0;
+ }
+
+ base_addr = be64_to_cpu(base_addr_be64);
+
+ rc = check_hotplug_memory_addressable(base_addr >> PAGE_SHIFT,
+ size >> PAGE_SHIFT);
+ if (rc)
+ return 0;
+
+ return base_addr;
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_platform_lpc_setup);
+
+void pnv_ocxl_platform_lpc_release(struct pci_dev *pdev)
+{
+ struct pci_controller *hose = pci_bus_to_host(pdev->bus);
+ struct pnv_phb *phb = hose->private_data;
+ u32 bdfn = pci_dev_id(pdev);
+ int rc;
+
+ rc = opal_npu_mem_release(phb->opal_id, bdfn);
+ if (rc)
+ dev_warn(&pdev->dev,
+"OPAL reported rc=%d when releasing LPC memory\n", rc);
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_platform_lpc_release);
+#endif
+
int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle)
{
struct spa_data *data = (struct spa_data *) platform_data;
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 05/27] ocxl: Address kernel doc errors & warnings
From: Alastair D'Silva
This patch addresses warnings and errors from the kernel doc scripts for
the OpenCAPI driver.
It also makes minor tweaks to make the docs more consistent.
Signed-off-by: Alastair D'Silva
---
drivers/misc/ocxl/config.c| 24
drivers/misc/ocxl/ocxl_internal.h | 9 +--
include/misc/ocxl.h | 96 ---
3 files changed, 55 insertions(+), 74 deletions(-)
diff --git a/drivers/misc/ocxl/config.c b/drivers/misc/ocxl/config.c
index c8e19bfb5ef9..a62e3d7db2bf 100644
--- a/drivers/misc/ocxl/config.c
+++ b/drivers/misc/ocxl/config.c
@@ -273,16 +273,16 @@ static int read_afu_info(struct pci_dev *dev, struct
ocxl_fn_config *fn,
}
/**
- * Read the template version from the AFU
- * dev: the device for the AFU
- * fn: the AFU offsets
- * len: outputs the template length
- * version: outputs the major<<8,minor version
+ * read_template_version() - Read the template version from the AFU
+ * @dev: the device for the AFU
+ * @fn: the AFU offsets
+ * @len: outputs the template length
+ * @version: outputs the major<<8,minor version
*
* Returns 0 on success, negative on failure
*/
static int read_template_version(struct pci_dev *dev, struct ocxl_fn_config
*fn,
- u16 *len, u16 *version)
+u16 *len, u16 *version)
{
u32 val32;
u8 major, minor;
@@ -476,16 +476,16 @@ static int validate_afu(struct pci_dev *dev, struct
ocxl_afu_config *afu)
}
/**
- * Populate AFU metadata regarding LPC memory
- * dev: the device for the AFU
- * fn: the AFU offsets
- * afu: the AFU struct to populate the LPC metadata into
+ * read_afu_lpc_memory_info() - Populate AFU metadata regarding LPC memory
+ * @dev: the device for the AFU
+ * @fn: the AFU offsets
+ * @afu: the AFU struct to populate the LPC metadata into
*
* Returns 0 on success, negative on failure
*/
static int read_afu_lpc_memory_info(struct pci_dev *dev,
- struct ocxl_fn_config *fn,
- struct ocxl_afu_config *afu)
+ struct ocxl_fn_config *fn,
+ struct ocxl_afu_config *afu)
{
int rc;
u32 val32;
diff --git a/drivers/misc/ocxl/ocxl_internal.h
b/drivers/misc/ocxl/ocxl_internal.h
index 345bf843a38e..198e4e4bc51d 100644
--- a/drivers/misc/ocxl/ocxl_internal.h
+++ b/drivers/misc/ocxl/ocxl_internal.h
@@ -122,11 +122,12 @@ int ocxl_config_check_afu_index(struct pci_dev *dev,
struct ocxl_fn_config *fn, int afu_idx);
/**
- * Update values within a Process Element
+ * ocxl_link_update_pe() - Update values within a Process Element
+ * @link_handle: the link handle associated with the process element
+ * @pasid: the PASID for the AFU context
+ * @tid: the new thread id for the process element
*
- * link_handle: the link handle associated with the process element
- * pasid: the PASID for the AFU context
- * tid: the new thread id for the process element
+ * Returns 0 on success
*/
int ocxl_link_update_pe(void *link_handle, int pasid, __u16 tid);
diff --git a/include/misc/ocxl.h b/include/misc/ocxl.h
index 0a762e387418..357ef1aadbc0 100644
--- a/include/misc/ocxl.h
+++ b/include/misc/ocxl.h
@@ -62,8 +62,7 @@ struct ocxl_context;
// Device detection & initialisation
/**
- * Open an OpenCAPI function on an OpenCAPI device
- *
+ * ocxl_function_open() - Open an OpenCAPI function on an OpenCAPI device
* @dev: The PCI device that contains the function
*
* Returns an opaque pointer to the function, or an error pointer (check with
IS_ERR)
@@ -71,8 +70,7 @@ struct ocxl_context;
struct ocxl_fn *ocxl_function_open(struct pci_dev *dev);
/**
- * Get the list of AFUs associated with a PCI function device
- *
+ * ocxl_function_afu_list() - Get the list of AFUs associated with a PCI
function device
* Returns a list of struct ocxl_afu *
*
* @fn: The OpenCAPI function containing the AFUs
@@ -80,8 +78,7 @@ struct ocxl_fn *ocxl_function_open(struct pci_dev *dev);
struct list_head *ocxl_function_afu_list(struct ocxl_fn *fn);
/**
- * Fetch an AFU instance from an OpenCAPI function
- *
+ * ocxl_function_fetch_afu() - Fetch an AFU instance from an OpenCAPI function
* @fn: The OpenCAPI function to get the AFU from
* @afu_idx: The index of the AFU to get
*
@@ -92,23 +89,20 @@ struct list_head *ocxl_function_afu_list(struct ocxl_fn
*fn);
struct ocxl_afu *ocxl_function_fetch_afu(struct ocxl_fn *fn, u8 afu_idx);
/**
- * Take a reference to an AFU
- *
+ * ocxl_afu_get() - Take a reference to an AFU
* @afu: The AFU to increment the reference count on
*/
void ocxl_afu_get(struct ocxl_afu *afu);
/**
- * Release a reference to an AFU
- *
+ * ocxl_afu_put() - Release a reference to an AFU
* @afu: The AFU to decrement the reference count on
*/
void ocxl_afu_put(struct ocxl_afu *afu);
/**
- * Get the config
[PATCH v3 10/27] powerpc: Add driver for OpenCAPI Persistent Memory
From: Alastair D'Silva
This driver exposes LPC memory on OpenCAPI pmem cards
as an NVDIMM, allowing the existing nvram infrastructure
to be used.
Namespace metadata is stored on the media itself, so
scm_reserve_metadata() maps 1 section's worth of PMEM storage
at the start to hold this. The rest of the PMEM range is registered
with libnvdimm as an nvdimm. scm_ndctl_config_read/write/size() provide
callbacks to libnvdimm to access the metadata.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/Kconfig| 3 +
arch/powerpc/platforms/powernv/Makefile | 1 +
arch/powerpc/platforms/powernv/pmem/Kconfig | 15 +
arch/powerpc/platforms/powernv/pmem/Makefile | 7 +
arch/powerpc/platforms/powernv/pmem/ocxl.c| 473 ++
.../platforms/powernv/pmem/ocxl_internal.h| 28 ++
6 files changed, 527 insertions(+)
create mode 100644 arch/powerpc/platforms/powernv/pmem/Kconfig
create mode 100644 arch/powerpc/platforms/powernv/pmem/Makefile
create mode 100644 arch/powerpc/platforms/powernv/pmem/ocxl.c
create mode 100644 arch/powerpc/platforms/powernv/pmem/ocxl_internal.h
diff --git a/arch/powerpc/platforms/powernv/Kconfig
b/arch/powerpc/platforms/powernv/Kconfig
index 938803eab0ad..fc8976af0e52 100644
--- a/arch/powerpc/platforms/powernv/Kconfig
+++ b/arch/powerpc/platforms/powernv/Kconfig
@@ -50,3 +50,6 @@ config PPC_VAS
config SCOM_DEBUGFS
bool "Expose SCOM controllers via debugfs"
depends on DEBUG_FS
+
+source "arch/powerpc/platforms/powernv/pmem/Kconfig"
+
diff --git a/arch/powerpc/platforms/powernv/Makefile
b/arch/powerpc/platforms/powernv/Makefile
index c0f8120045c3..0bbd72988b6f 100644
--- a/arch/powerpc/platforms/powernv/Makefile
+++ b/arch/powerpc/platforms/powernv/Makefile
@@ -21,3 +21,4 @@ obj-$(CONFIG_PPC_VAS) += vas.o vas-window.o vas-debug.o
obj-$(CONFIG_OCXL_BASE)+= ocxl.o
obj-$(CONFIG_SCOM_DEBUGFS) += opal-xscom.o
obj-$(CONFIG_PPC_SECURE_BOOT) += opal-secvar.o
+obj-$(CONFIG_LIBNVDIMM) += pmem/
diff --git a/arch/powerpc/platforms/powernv/pmem/Kconfig
b/arch/powerpc/platforms/powernv/pmem/Kconfig
new file mode 100644
index ..c5d927520920
--- /dev/null
+++ b/arch/powerpc/platforms/powernv/pmem/Kconfig
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0-only
+if LIBNVDIMM
+
+config OCXL_PMEM
+ tristate "OpenCAPI Persistent Memory"
+ depends on LIBNVDIMM && PPC_POWERNV && PCI && EEH && ZONE_DEVICE && OCXL
+ help
+ Exposes devices that implement the OpenCAPI Storage Class Memory
+ specification as persistent memory regions. You may also want
+ DEV_DAX, DEV_DAX_PMEM & FS_DAX if you plan on using DAX devices
+ stacked on top of this driver.
+
+ Select N if unsure.
+
+endif
diff --git a/arch/powerpc/platforms/powernv/pmem/Makefile
b/arch/powerpc/platforms/powernv/pmem/Makefile
new file mode 100644
index ..1c55c4193175
--- /dev/null
+++ b/arch/powerpc/platforms/powernv/pmem/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+
+ccflags-$(CONFIG_PPC_WERROR) += -Werror
+
+obj-$(CONFIG_OCXL_PMEM) += ocxlpmem.o
+
+ocxlpmem-y := ocxl.o
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
new file mode 100644
index ..3c4eeb5dcc0f
--- /dev/null
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -0,0 +1,473 @@
+// SPDX-License-Id
+// Copyright 2019 IBM Corp.
+
+/*
+ * A driver for OpenCAPI devices that implement the Storage Class
+ * Memory specification.
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include "ocxl_internal.h"
+
+
+static const struct pci_device_id ocxlpmem_pci_tbl[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0625), },
+ { }
+};
+
+MODULE_DEVICE_TABLE(pci, ocxlpmem_pci_tbl);
+
+#define NUM_MINORS 256 // Total to reserve
+
+static dev_t ocxlpmem_dev;
+static struct class *ocxlpmem_class;
+static struct mutex minors_idr_lock;
+static struct idr minors_idr;
+
+/**
+ * ndctl_config_write() - Handle a ND_CMD_SET_CONFIG_DATA command from ndctl
+ * @ocxlpmem: the device metadata
+ * @command: the incoming data to write
+ * Return: 0 on success, negative on failure
+ */
+static int ndctl_config_write(struct ocxlpmem *ocxlpmem,
+ struct nd_cmd_set_config_hdr *command)
+{
+ if (command->in_offset + command->in_length > LABEL_AREA_SIZE)
+ return -EINVAL;
+
+ memcpy_flushcache(ocxlpmem->metadata_addr + command->in_offset,
command->in_buf,
+ command->in_length);
+
+ return 0;
+}
+
+/**
+ * ndctl_config_read() - Handle a ND_CMD_GET_CONFIG_DATA command from ndctl
+ * @ocxlpmem: the device metadata
+ * @command: the read request
+ * Return: 0 on success, negative on failure
+ */
+static int ndctl_config_read(struct ocxlpmem *ocxlpmem,
+
[PATCH v3 20/27] powerpc/powernv/pmem: Forward events to userspace
From: Alastair D'Silva
Some of the interrupts that the card generates are better handled
by the userspace daemon, in particular:
Controller Hardware/Firmware Fatal
Controller Dump Available
Error Log available
This patch allows a userspace application to register an eventfd with
the driver via SCM_IOCTL_EVENTFD to receive notifications of these
interrupts.
Userspace can then identify what events have occurred by calling
SCM_IOCTL_EVENT_CHECK and checking against the SCM_IOCTL_EVENT_FOO
masks.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/ocxl.c| 216 ++
.../platforms/powernv/pmem/ocxl_internal.h| 5 +
include/uapi/nvdimm/ocxl-pmem.h | 16 ++
3 files changed, 237 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index 009d4fd29e7d..e46696d3cc36 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -10,6 +10,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -335,11 +336,22 @@ static void free_ocxlpmem(struct ocxlpmem *ocxlpmem)
{
int rc;
+ // Disable doorbells
+ (void)ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CHIEC,
+OCXL_LITTLE_ENDIAN,
+GLOBAL_MMIO_CHI_ALL);
+
if (ocxlpmem->nvdimm_bus)
nvdimm_bus_unregister(ocxlpmem->nvdimm_bus);
free_minor(ocxlpmem);
+ if (ocxlpmem->irq_addr[1])
+ iounmap(ocxlpmem->irq_addr[1]);
+
+ if (ocxlpmem->irq_addr[0])
+ iounmap(ocxlpmem->irq_addr[0]);
+
if (ocxlpmem->cdev.owner)
cdev_del(&ocxlpmem->cdev);
@@ -443,6 +455,11 @@ static int file_release(struct inode *inode, struct file
*file)
{
struct ocxlpmem *ocxlpmem = file->private_data;
+ if (ocxlpmem->ev_ctx) {
+ eventfd_ctx_put(ocxlpmem->ev_ctx);
+ ocxlpmem->ev_ctx = NULL;
+ }
+
ocxlpmem_put(ocxlpmem);
return 0;
}
@@ -938,6 +955,51 @@ static int ioctl_controller_stats(struct ocxlpmem
*ocxlpmem,
return rc;
}
+static int ioctl_eventfd(struct ocxlpmem *ocxlpmem,
+struct ioctl_ocxl_pmem_eventfd __user *uarg)
+{
+ struct ioctl_ocxl_pmem_eventfd args;
+
+ if (copy_from_user(&args, uarg, sizeof(args)))
+ return -EFAULT;
+
+ if (ocxlpmem->ev_ctx)
+ return -EINVAL;
+
+ ocxlpmem->ev_ctx = eventfd_ctx_fdget(args.eventfd);
+ if (!ocxlpmem->ev_ctx)
+ return -EFAULT;
+
+ return 0;
+}
+
+static int ioctl_event_check(struct ocxlpmem *ocxlpmem, u64 __user *uarg)
+{
+ u64 val = 0;
+ int rc;
+ u64 chi = 0;
+
+ rc = ocxlpmem_chi(ocxlpmem, &chi);
+ if (rc < 0)
+ return rc;
+
+ if (chi & GLOBAL_MMIO_CHI_ELA)
+ val |= IOCTL_OCXL_PMEM_EVENT_ERROR_LOG_AVAILABLE;
+
+ if (chi & GLOBAL_MMIO_CHI_CDA)
+ val |= IOCTL_OCXL_PMEM_EVENT_CONTROLLER_DUMP_AVAILABLE;
+
+ if (chi & GLOBAL_MMIO_CHI_CFFS)
+ val |= IOCTL_OCXL_PMEM_EVENT_FIRMWARE_FATAL;
+
+ if (chi & GLOBAL_MMIO_CHI_CHFS)
+ val |= IOCTL_OCXL_PMEM_EVENT_HARDWARE_FATAL;
+
+ rc = copy_to_user((u64 __user *) uarg, &val, sizeof(val));
+
+ return rc;
+}
+
static long file_ioctl(struct file *file, unsigned int cmd, unsigned long args)
{
struct ocxlpmem *ocxlpmem = file->private_data;
@@ -966,6 +1028,15 @@ static long file_ioctl(struct file *file, unsigned int
cmd, unsigned long args)
rc = ioctl_controller_stats(ocxlpmem,
(struct
ioctl_ocxl_pmem_controller_stats __user *)args);
break;
+
+ case IOCTL_OCXL_PMEM_EVENTFD:
+ rc = ioctl_eventfd(ocxlpmem,
+ (struct ioctl_ocxl_pmem_eventfd __user
*)args);
+ break;
+
+ case IOCTL_OCXL_PMEM_EVENT_CHECK:
+ rc = ioctl_event_check(ocxlpmem, (u64 __user *)args);
+ break;
}
return rc;
@@ -1107,6 +1178,146 @@ static void dump_error_log(struct ocxlpmem *ocxlpmem)
kfree(buf);
}
+static irqreturn_t imn0_handler(void *private)
+{
+ struct ocxlpmem *ocxlpmem = private;
+ u64 chi = 0;
+
+ (void)ocxlpmem_chi(ocxlpmem, &chi);
+
+ if (chi & GLOBAL_MMIO_CHI_ELA) {
+ dev_warn(&ocxlpmem->dev, "Error log is available\n");
+
+ if (ocxlpmem->ev_ctx)
+ eventfd_signal(ocxlpmem->ev_ctx, 1);
+ }
+
+ if (chi & GLOBAL_MMIO_CHI_CDA) {
+ dev_warn(&ocxlpmem->dev, "Controller dump is available\n");
+
+
[PATCH v3 15/27] powerpc/powernv/pmem: Add support for near storage commands
From: Alastair D'Silva
Similar to the previous patch, this adds support for near storage commands.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/ocxl.c| 6 +++
.../platforms/powernv/pmem/ocxl_internal.c| 41 +++
.../platforms/powernv/pmem/ocxl_internal.h| 37 +
3 files changed, 84 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index 4e782d22605b..b8bd7e703b19 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -259,12 +259,18 @@ static int setup_command_metadata(struct ocxlpmem
*ocxlpmem)
int rc;
mutex_init(&ocxlpmem->admin_command.lock);
+ mutex_init(&ocxlpmem->ns_command.lock);
rc = extract_command_metadata(ocxlpmem, GLOBAL_MMIO_ACMA_CREQO,
&ocxlpmem->admin_command);
if (rc)
return rc;
+ rc = extract_command_metadata(ocxlpmem, GLOBAL_MMIO_NSCMA_CREQO,
+ &ocxlpmem->ns_command);
+ if (rc)
+ return rc;
+
return 0;
}
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
index 583f48023025..3e0b133feddf 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
@@ -133,6 +133,47 @@ int admin_response_handled(const struct ocxlpmem *ocxlpmem)
OCXL_LITTLE_ENDIAN, GLOBAL_MMIO_CHI_ACRA);
}
+int ns_command_request(struct ocxlpmem *ocxlpmem, u8 op_code)
+{
+ u64 val;
+ int rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CHI,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ return rc;
+
+ if (!(val & GLOBAL_MMIO_CHI_NSCRA))
+ return -EBUSY;
+
+ return scm_command_request(ocxlpmem, &ocxlpmem->ns_command, op_code);
+}
+
+int ns_response(const struct ocxlpmem *ocxlpmem)
+{
+ return command_response(ocxlpmem, &ocxlpmem->ns_command);
+}
+
+int ns_command_execute(const struct ocxlpmem *ocxlpmem)
+{
+ return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_HCI,
+ OCXL_LITTLE_ENDIAN,
GLOBAL_MMIO_HCI_NSCRW);
+}
+
+bool ns_command_complete(const struct ocxlpmem *ocxlpmem)
+{
+ u64 val = 0;
+ int rc = ocxlpmem_chi(ocxlpmem, &val);
+
+ WARN_ON(rc);
+
+ return (val & GLOBAL_MMIO_CHI_NSCRA) != 0;
+}
+
+int ns_response_handled(const struct ocxlpmem *ocxlpmem)
+{
+ return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CHIC,
+ OCXL_LITTLE_ENDIAN,
GLOBAL_MMIO_CHI_NSCRA);
+}
+
void warn_status(const struct ocxlpmem *ocxlpmem, const char *message,
u8 status)
{
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.h
b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.h
index 2fef68c71271..28e2020f6355 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.h
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.h
@@ -107,6 +107,7 @@ struct ocxlpmem {
struct ocxl_context *ocxl_context;
void *metadata_addr;
struct command_metadata admin_command;
+ struct command_metadata ns_command;
struct resource pmem_res;
struct nd_region *nd_region;
char fw_version[8+1];
@@ -175,6 +176,42 @@ int admin_command_complete_timeout(const struct ocxlpmem
*ocxlpmem,
*/
int admin_response_handled(const struct ocxlpmem *ocxlpmem);
+/**
+ * ns_command_request() - Issue a near storage command request
+ * @ocxlpmem: the device metadata
+ * @op_code: The op-code for the command
+ * Returns an identifier for the command, or negative on error
+ */
+int ns_command_request(struct ocxlpmem *ocxlpmem, u8 op_code);
+
+/**
+ * ns_response() - Validate a near storage response
+ * @ocxlpmem: the device metadata
+ * Returns the status code of the command, or negative on error
+ */
+int ns_response(const struct ocxlpmem *ocxlpmem);
+
+/**
+ * ns_command_execute() - Notify the controller to start processing a pending
near storage command
+ * @ocxlpmem: the device metadata
+ * Returns 0 on success, negative on error
+ */
+int ns_command_execute(const struct ocxlpmem *ocxlpmem);
+
+/**
+ * ns_command_complete() - Is a near storage command executing
+ * @ocxlpmem: the device metadata
+ * Returns true if the previous admin command has completed
+ */
+bool ns_command_complete(const struct ocxlpmem *ocxlpmem);
+
+/**
+ * ns_response_handled() - Notify the controller that the near storage
response has been handled
+ * @ocxlpmem: the device metadata
+ * Returns 0 on success, negative on failure
+ */
+int ns_response_handled(const struct oc
[PATCH v3 06/27] ocxl: Tally up the LPC memory on a link & allow it to be mapped
From: Alastair D'Silva
Tally up the LPC memory on an OpenCAPI link & allow it to be mapped
Signed-off-by: Alastair D'Silva
---
drivers/misc/ocxl/core.c | 10 ++
drivers/misc/ocxl/link.c | 53 +++
drivers/misc/ocxl/ocxl_internal.h | 33 +++
3 files changed, 96 insertions(+)
diff --git a/drivers/misc/ocxl/core.c b/drivers/misc/ocxl/core.c
index b7a09b21ab36..2531c6cf19a0 100644
--- a/drivers/misc/ocxl/core.c
+++ b/drivers/misc/ocxl/core.c
@@ -230,8 +230,18 @@ static int configure_afu(struct ocxl_afu *afu, u8 afu_idx,
struct pci_dev *dev)
if (rc)
goto err_free_pasid;
+ if (afu->config.lpc_mem_size || afu->config.special_purpose_mem_size) {
+ rc = ocxl_link_add_lpc_mem(afu->fn->link,
afu->config.lpc_mem_offset,
+ afu->config.lpc_mem_size +
+
afu->config.special_purpose_mem_size);
+ if (rc)
+ goto err_free_mmio;
+ }
+
return 0;
+err_free_mmio:
+ unmap_mmio_areas(afu);
err_free_pasid:
reclaim_afu_pasid(afu);
err_free_actag:
diff --git a/drivers/misc/ocxl/link.c b/drivers/misc/ocxl/link.c
index 58d111afd9f6..1e039cc5ebe5 100644
--- a/drivers/misc/ocxl/link.c
+++ b/drivers/misc/ocxl/link.c
@@ -84,6 +84,11 @@ struct ocxl_link {
int dev;
atomic_t irq_available;
struct spa *spa;
+ struct mutex lpc_mem_lock; /* protects lpc_mem & lpc_mem_sz */
+ u64 lpc_mem_sz; /* Total amount of LPC memory presented on the link */
+ u64 lpc_mem;
+ int lpc_consumers;
+
void *platform_data;
};
static struct list_head links_list = LIST_HEAD_INIT(links_list);
@@ -396,6 +401,8 @@ static int alloc_link(struct pci_dev *dev, int PE_mask,
struct ocxl_link **out_l
if (rc)
goto err_spa;
+ mutex_init(&link->lpc_mem_lock);
+
/* platform specific hook */
rc = pnv_ocxl_spa_setup(dev, link->spa->spa_mem, PE_mask,
&link->platform_data);
@@ -711,3 +718,49 @@ void ocxl_link_free_irq(void *link_handle, int hw_irq)
atomic_inc(&link->irq_available);
}
EXPORT_SYMBOL_GPL(ocxl_link_free_irq);
+
+int ocxl_link_add_lpc_mem(void *link_handle, u64 offset, u64 size)
+{
+ struct ocxl_link *link = (struct ocxl_link *) link_handle;
+
+ // Check for overflow
+ if (offset > (offset + size))
+ return -EINVAL;
+
+ mutex_lock(&link->lpc_mem_lock);
+ link->lpc_mem_sz = max(link->lpc_mem_sz, offset + size);
+
+ mutex_unlock(&link->lpc_mem_lock);
+
+ return 0;
+}
+
+u64 ocxl_link_lpc_map(void *link_handle, struct pci_dev *pdev)
+{
+ struct ocxl_link *link = (struct ocxl_link *) link_handle;
+
+ mutex_lock(&link->lpc_mem_lock);
+
+ if(!link->lpc_mem)
+ link->lpc_mem = pnv_ocxl_platform_lpc_setup(pdev,
link->lpc_mem_sz);
+
+ if(link->lpc_mem)
+ link->lpc_consumers++;
+ mutex_unlock(&link->lpc_mem_lock);
+
+ return link->lpc_mem;
+}
+
+void ocxl_link_lpc_release(void *link_handle, struct pci_dev *pdev)
+{
+ struct ocxl_link *link = (struct ocxl_link *) link_handle;
+
+ mutex_lock(&link->lpc_mem_lock);
+ WARN_ON(--link->lpc_consumers < 0);
+ if (link->lpc_consumers == 0) {
+ pnv_ocxl_platform_lpc_release(pdev);
+ link->lpc_mem = 0;
+ }
+
+ mutex_unlock(&link->lpc_mem_lock);
+}
diff --git a/drivers/misc/ocxl/ocxl_internal.h
b/drivers/misc/ocxl/ocxl_internal.h
index 198e4e4bc51d..d0c8c4838f42 100644
--- a/drivers/misc/ocxl/ocxl_internal.h
+++ b/drivers/misc/ocxl/ocxl_internal.h
@@ -142,4 +142,37 @@ int ocxl_irq_offset_to_id(struct ocxl_context *ctx, u64
offset);
u64 ocxl_irq_id_to_offset(struct ocxl_context *ctx, int irq_id);
void ocxl_afu_irq_free_all(struct ocxl_context *ctx);
+/**
+ * ocxl_link_add_lpc_mem() - Increment the amount of memory required by an
OpenCAPI link
+ *
+ * @link_handle: The OpenCAPI link handle
+ * @offset: The offset of the memory to add
+ * @size: The amount of memory to increment by
+ *
+ * Returns 0 on success, negative on overflow
+ */
+int ocxl_link_add_lpc_mem(void *link_handle, u64 offset, u64 size);
+
+/**
+ * ocxl_link_lpc_map() - Map the LPC memory for an OpenCAPI device
+ * Since LPC memory belongs to a link, the whole LPC memory available
+ * on the link must be mapped in order to make it accessible to a device.
+ * @link_handle: The OpenCAPI link handle
+ * @pdev: A device that is on the link
+ *
+ * Returns the address of the mapped LPC memory, or 0 on error
+ */
+u64 ocxl_link_lpc_map(void *link_handle, struct pci_dev *pdev);
+
+/**
+ * ocxl_link_lpc_release() - Release the LPC memory device for
[PATCH v3 25/27] powerpc/powernv/pmem: Expose the serial number in sysfs
From: Alastair D'Silva
This information will be used by ndctl in userspace to help users identify
the device.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/Makefile | 2 +-
arch/powerpc/platforms/powernv/pmem/ocxl.c| 5 +++
.../platforms/powernv/pmem/ocxl_internal.h| 6 +++
.../platforms/powernv/pmem/ocxl_sysfs.c | 37 +++
4 files changed, 49 insertions(+), 1 deletion(-)
create mode 100644 arch/powerpc/platforms/powernv/pmem/ocxl_sysfs.c
diff --git a/arch/powerpc/platforms/powernv/pmem/Makefile
b/arch/powerpc/platforms/powernv/pmem/Makefile
index 4ceda25907d4..d02870806f30 100644
--- a/arch/powerpc/platforms/powernv/pmem/Makefile
+++ b/arch/powerpc/platforms/powernv/pmem/Makefile
@@ -4,4 +4,4 @@ ccflags-$(CONFIG_PPC_WERROR)+= -Werror
obj-$(CONFIG_OCXL_PMEM) += ocxlpmem.o
-ocxlpmem-y := ocxl.o ocxl_internal.o
+ocxlpmem-y := ocxl.o ocxl_internal.o ocxl_sysfs.o
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index 5cd1b6d78dd6..ec73713d05ad 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -1878,6 +1878,11 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
goto err;
}
+ if (ocxlpmem_sysfs_add(ocxlpmem)) {
+ dev_err(&pdev->dev, "Could not create sysfs entries\n");
+ goto err;
+ }
+
elapsed = 0;
timeout = ocxlpmem->readiness_timeout +
ocxlpmem->memory_available_timeout;
while (!is_usable(ocxlpmem, false)) {
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.h
b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.h
index 0eb7a35d24ae..12304ceace61 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.h
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.h
@@ -246,3 +246,9 @@ int ns_response_handled(const struct ocxlpmem *ocxlpmem);
*/
void warn_status(const struct ocxlpmem *ocxlpmem, const char *message,
u8 status);
+
+/**
+ * ocxlpmem_sysfs_add() - Create sysfs entries for an OpenCAPI persistent
memory device
+ * @ocxlpmem: the device metadata
+ */
+int ocxlpmem_sysfs_add(struct ocxlpmem *ocxlpmem);
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl_sysfs.c
b/arch/powerpc/platforms/powernv/pmem/ocxl_sysfs.c
new file mode 100644
index ..7829e4bc887d
--- /dev/null
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl_sysfs.c
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2018 IBM Corp.
+
+#include
+#include
+#include
+#include
+#include "ocxl_internal.h"
+
+static ssize_t serial_show(struct device *device, struct device_attribute
*attr,
+ char *buf)
+{
+ struct ocxlpmem *ocxlpmem = container_of(device, struct ocxlpmem, dev);
+ const struct ocxl_fn_config *fn_config =
ocxl_function_config(ocxlpmem->ocxl_fn);
+
+ return scnprintf(buf, PAGE_SIZE, "%llu\n", fn_config->serial);
+}
+
+static struct device_attribute attrs[] = {
+ __ATTR_RO(serial),
+};
+
+int ocxlpmem_sysfs_add(struct ocxlpmem *ocxlpmem)
+{
+ int i, rc;
+
+ for (i = 0; i < ARRAY_SIZE(attrs); i++) {
+ rc = device_create_file(&ocxlpmem->dev, &attrs[i]);
+ if (rc) {
+ for (; --i >= 0;)
+ device_remove_file(&ocxlpmem->dev, &attrs[i]);
+
+ return rc;
+ }
+ }
+ return 0;
+}
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 22/27] powerpc/powernv/pmem: Implement the heartbeat command
From: Alastair D'Silva
The heartbeat admin command is a simple admin command that exercises
the communication mechanisms within the controller.
This patch issues a heartbeat command to the card during init to ensure
we can communicate with the card's controller.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/ocxl.c | 43 ++
1 file changed, 43 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index 081883a8247a..e01f6f9fc180 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -306,6 +306,44 @@ static bool is_usable(const struct ocxlpmem *ocxlpmem,
bool verbose)
return true;
}
+/**
+ * heartbeat() - Issue a heartbeat command to the controller
+ * @ocxlpmem: the device metadata
+ * Return: 0 if the controller responded correctly, negative on error
+ */
+static int heartbeat(struct ocxlpmem *ocxlpmem)
+{
+ int rc;
+
+ mutex_lock(&ocxlpmem->admin_command.lock);
+
+ rc = admin_command_request(ocxlpmem, ADMIN_COMMAND_HEARTBEAT);
+ if (rc)
+ goto out;
+
+ rc = admin_command_execute(ocxlpmem);
+ if (rc)
+ goto out;
+
+ rc = admin_command_complete_timeout(ocxlpmem, ADMIN_COMMAND_HEARTBEAT);
+ if (rc < 0) {
+ dev_err(&ocxlpmem->dev, "Heartbeat timeout\n");
+ goto out;
+ }
+
+ rc = admin_response(ocxlpmem);
+ if (rc < 0)
+ goto out;
+ if (rc != STATUS_SUCCESS)
+ warn_status(ocxlpmem, "Unexpected status from heartbeat", rc);
+
+ (void)admin_response_handled(ocxlpmem);
+
+out:
+ mutex_unlock(&ocxlpmem->admin_command.lock);
+ return rc;
+}
+
/**
* allocate_minor() - Allocate a minor number to use for an OpenCAPI pmem
device
* @ocxlpmem: the device metadata
@@ -1458,6 +1496,11 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
goto err;
}
+ if (heartbeat(ocxlpmem)) {
+ dev_err(&pdev->dev, "Heartbeat failed\n");
+ goto err;
+ }
+
elapsed = 0;
timeout = ocxlpmem->readiness_timeout +
ocxlpmem->memory_available_timeout;
while (!is_usable(ocxlpmem, false)) {
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 18/27] powerpc/powernv/pmem: Add controller dump IOCTLs
From: Alastair D'Silva
This patch adds IOCTLs to allow userspace to request & fetch dumps
of the internal controller state.
This is useful during debugging or when a fatal error on the controller
has occurred.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/ocxl.c | 132 +
include/uapi/nvdimm/ocxl-pmem.h| 15 +++
2 files changed, 147 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index 2b64504f9129..2cabafe1fc58 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -640,6 +640,124 @@ static int ioctl_error_log(struct ocxlpmem *ocxlpmem,
return 0;
}
+static int ioctl_controller_dump_data(struct ocxlpmem *ocxlpmem,
+ struct ioctl_ocxl_pmem_controller_dump_data __user *uarg)
+{
+ struct ioctl_ocxl_pmem_controller_dump_data args;
+ u16 i;
+ u64 val;
+ int rc;
+
+ if (copy_from_user(&args, uarg, sizeof(args)))
+ return -EFAULT;
+
+ if (args.buf_size % 8)
+ return -EINVAL;
+
+ if (args.buf_size > ocxlpmem->admin_command.data_size)
+ return -EINVAL;
+
+ mutex_lock(&ocxlpmem->admin_command.lock);
+
+ rc = admin_command_request(ocxlpmem, ADMIN_COMMAND_CONTROLLER_DUMP);
+ if (rc)
+ goto out;
+
+ val = ((u64)args.offset) << 32;
+ val |= args.buf_size;
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+ ocxlpmem->admin_command.request_offset +
0x08,
+ OCXL_LITTLE_ENDIAN, val);
+ if (rc)
+ goto out;
+
+ rc = admin_command_execute(ocxlpmem);
+ if (rc)
+ goto out;
+
+ rc = admin_command_complete_timeout(ocxlpmem,
+ ADMIN_COMMAND_CONTROLLER_DUMP);
+ if (rc < 0) {
+ dev_warn(&ocxlpmem->dev, "Controller dump timed out\n");
+ goto out;
+ }
+
+ rc = admin_response(ocxlpmem);
+ if (rc < 0)
+ goto out;
+ if (rc != STATUS_SUCCESS) {
+ warn_status(ocxlpmem,
+ "Unexpected status from retrieve error log",
+ rc);
+ goto out;
+ }
+
+ for (i = 0; i < args.buf_size; i += 8) {
+ u64 val;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+
ocxlpmem->admin_command.data_offset + i,
+OCXL_HOST_ENDIAN, &val);
+ if (rc)
+ goto out;
+
+ if (copy_to_user(&args.buf[i], &val, sizeof(u64))) {
+ rc = -EFAULT;
+ goto out;
+ }
+ }
+
+ if (copy_to_user(uarg, &args, sizeof(args))) {
+ rc = -EFAULT;
+ goto out;
+ }
+
+ rc = admin_response_handled(ocxlpmem);
+ if (rc)
+ goto out;
+
+out:
+ mutex_unlock(&ocxlpmem->admin_command.lock);
+ return rc;
+}
+
+int request_controller_dump(struct ocxlpmem *ocxlpmem)
+{
+ int rc;
+ u64 busy = 1;
+
+ rc = ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CHIC,
+ OCXL_LITTLE_ENDIAN,
+ GLOBAL_MMIO_CHI_CDA);
+
+
+ rc = ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_HCI,
+ OCXL_LITTLE_ENDIAN,
+ GLOBAL_MMIO_HCI_CONTROLLER_DUMP);
+ if (rc)
+ return rc;
+
+ while (busy) {
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
+GLOBAL_MMIO_HCI,
+OCXL_LITTLE_ENDIAN, &busy);
+ if (rc)
+ return rc;
+
+ busy &= GLOBAL_MMIO_HCI_CONTROLLER_DUMP;
+ cond_resched();
+ }
+
+ return 0;
+}
+
+static int ioctl_controller_dump_complete(struct ocxlpmem *ocxlpmem)
+{
+ return ocxl_global_mmio_set64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_HCI,
+ OCXL_LITTLE_ENDIAN,
+ GLOBAL_MMIO_HCI_CONTROLLER_DUMP_COLLECTED);
+}
+
static long file_ioctl(struct file *file, unsigned int cmd, unsigned long args)
{
struct ocxlpmem *ocxlpmem = file->private_data;
@@ -650,7 +768,21 @@ static long file_ioctl(struct file *file, unsigned int
cmd, unsigned long args)
rc = ioctl_error_log(ocxlpmem,
(struct ioctl_ocxl_pmem_error_log __user
*)args);
b
[PATCH v3 27/27] MAINTAINERS: Add myself & nvdimm/ocxl to ocxl
From: Alastair D'Silva The OpenCAPI Persistent Memory driver will be maintained as part ofi the ppc tree. I'm also adding myself as an author of the driver & contributor to the generic ocxl driver. Signed-off-by: Alastair D'Silva --- MAINTAINERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index f8670989ec91..3fb9a9f576a7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12064,13 +12064,16 @@ F:tools/objtool/ OCXL (Open Coherent Accelerator Processor Interface OpenCAPI) DRIVER M: Frederic Barrat M: Andrew Donnellan +M: Alastair D'Silva L: linuxppc-...@lists.ozlabs.org S: Supported F: arch/powerpc/platforms/powernv/ocxl.c +F: arch/powerpc/platforms/powernv/pmem/* F: arch/powerpc/include/asm/pnv-ocxl.h F: drivers/misc/ocxl/ F: include/misc/ocxl* F: include/uapi/misc/ocxl.h +F: include/uapi/nvdimm/ocxl-pmem.h F: Documentation/userspace-api/accelerators/ocxl.rst OMAP AUDIO SUPPORT -- 2.24.1 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 13/27] powerpc/powernv/pmem: Read the capability registers & wait for device ready
From: Alastair D'Silva
This patch reads timeouts & firmware version from the controller, and
uses those timeouts to wait for the controller to report that it is ready
before handing the memory over to libnvdimm.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/Makefile | 2 +-
arch/powerpc/platforms/powernv/pmem/ocxl.c| 92 +++
.../platforms/powernv/pmem/ocxl_internal.c| 19
.../platforms/powernv/pmem/ocxl_internal.h| 24 +
4 files changed, 136 insertions(+), 1 deletion(-)
create mode 100644 arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
diff --git a/arch/powerpc/platforms/powernv/pmem/Makefile
b/arch/powerpc/platforms/powernv/pmem/Makefile
index 1c55c4193175..4ceda25907d4 100644
--- a/arch/powerpc/platforms/powernv/pmem/Makefile
+++ b/arch/powerpc/platforms/powernv/pmem/Makefile
@@ -4,4 +4,4 @@ ccflags-$(CONFIG_PPC_WERROR)+= -Werror
obj-$(CONFIG_OCXL_PMEM) += ocxlpmem.o
-ocxlpmem-y := ocxl.o
+ocxlpmem-y := ocxl.o ocxl_internal.o
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index 3c4eeb5dcc0f..431212c9f0cc 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -8,6 +8,7 @@
#include
#include
+#include
#include
#include
#include
@@ -215,6 +216,36 @@ static int register_lpc_mem(struct ocxlpmem *ocxlpmem)
return 0;
}
+/**
+ * is_usable() - Is a controller usable?
+ * @ocxlpmem: the device metadata
+ * @verbose: True to log errors
+ * Return: true if the controller is usable
+ */
+static bool is_usable(const struct ocxlpmem *ocxlpmem, bool verbose)
+{
+ u64 chi = 0;
+ int rc = ocxlpmem_chi(ocxlpmem, &chi);
+
+ if (rc < 0)
+ return false;
+
+ if (!(chi & GLOBAL_MMIO_CHI_CRDY)) {
+ if (verbose)
+ dev_err(&ocxlpmem->dev, "controller is not ready.\n");
+ return false;
+ }
+
+ if (!(chi & GLOBAL_MMIO_CHI_MA)) {
+ if (verbose)
+ dev_err(&ocxlpmem->dev,
+ "controller does not have memory available.\n");
+ return false;
+ }
+
+ return true;
+}
+
/**
* allocate_minor() - Allocate a minor number to use for an OpenCAPI pmem
device
* @ocxlpmem: the device metadata
@@ -328,6 +359,48 @@ static void ocxlpmem_remove(struct pci_dev *pdev)
}
}
+/**
+ * read_device_metadata() - Retrieve config information from the AFU and save
it for future use
+ * @ocxlpmem: the device metadata
+ * Return: 0 on success, negative on failure
+ */
+static int read_device_metadata(struct ocxlpmem *ocxlpmem)
+{
+ u64 val;
+ int rc;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CCAP0,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ return rc;
+
+ ocxlpmem->scm_revision = val & 0x;
+ ocxlpmem->read_latency = (val >> 32) & 0xFF;
+ ocxlpmem->readiness_timeout = (val >> 48) & 0x0F;
+ ocxlpmem->memory_available_timeout = val >> 52;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_CCAP1,
+OCXL_LITTLE_ENDIAN, &val);
+ if (rc)
+ return rc;
+
+ ocxlpmem->max_controller_dump_size = val & 0x;
+
+ // Extract firmware version text
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, GLOBAL_MMIO_FWVER,
+OCXL_HOST_ENDIAN, (u64
*)ocxlpmem->fw_version);
+ if (rc)
+ return rc;
+
+ ocxlpmem->fw_version[8] = '\0';
+
+ dev_info(&ocxlpmem->dev,
+"Firmware version '%s' SCM revision %d:%d\n",
ocxlpmem->fw_version,
+ocxlpmem->scm_revision >> 4, ocxlpmem->scm_revision & 0x0F);
+
+ return 0;
+}
+
/**
* probe_function0() - Set up function 0 for an OpenCAPI persistent memory
device
* This is important as it enables templates higher than 0 across all other
functions,
@@ -368,6 +441,7 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
{
struct ocxlpmem *ocxlpmem;
int rc;
+ u16 elapsed, timeout;
if (PCI_FUNC(pdev->devfn) == 0)
return probe_function0(pdev);
@@ -422,6 +496,24 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
goto err;
}
+ if (read_device_metadata(ocxlpmem)) {
+ dev_err(&pdev->dev, "Could not read metadata\n");
+ goto err;
+ }
+
+ elapsed = 0;
+ timeout = ocxlpmem->readiness_timeout +
ocxlpmem->memory_available_timeout;
+ w
[PATCH v3 11/27] powerpc: Enable the OpenCAPI Persistent Memory driver for powernv_defconfig
From: Alastair D'Silva This patch enables the OpenCAPI Persistent Memory driver, as well as DAX support, for the 'powernv' platform. DAX is not a strict requirement for the functioning of the driver, but it is likely that a user will want to create a DAX device on top of their persistent memory device. Signed-off-by: Alastair D'Silva --- arch/powerpc/configs/powernv_defconfig | 5 + 1 file changed, 5 insertions(+) diff --git a/arch/powerpc/configs/powernv_defconfig b/arch/powerpc/configs/powernv_defconfig index 71749377d164..921d77bbd3d2 100644 --- a/arch/powerpc/configs/powernv_defconfig +++ b/arch/powerpc/configs/powernv_defconfig @@ -348,3 +348,8 @@ CONFIG_KVM_BOOK3S_64=m CONFIG_KVM_BOOK3S_64_HV=m CONFIG_VHOST_NET=m CONFIG_PRINTK_TIME=y +CONFIG_ZONE_DEVICE=y +CONFIG_OCXL_PMEM=m +CONFIG_DEV_DAX=m +CONFIG_DEV_DAX_PMEM=m +CONFIG_FS_DAX=y -- 2.24.1 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 00/27] Add support for OpenCAPI Persistent Memory devices
From: Alastair D'Silva This series adds support for OpenCAPI Persistent Memory devices, exposing them as nvdimms so that we can make use of the existing infrastructure. Alastair D'Silva (27): powerpc: Add OPAL calls for LPC memory alloc/release mm/memory_hotplug: Allow check_hotplug_memory_addressable to be called from drivers powerpc: Map & release OpenCAPI LPC memory ocxl: Remove unnecessary externs ocxl: Address kernel doc errors & warnings ocxl: Tally up the LPC memory on a link & allow it to be mapped ocxl: Add functions to map/unmap LPC memory ocxl: Emit a log message showing how much LPC memory was detected ocxl: Save the device serial number in ocxl_fn powerpc: Add driver for OpenCAPI Persistent Memory powerpc: Enable the OpenCAPI Persistent Memory driver for powernv_defconfig powerpc/powernv/pmem: Add register addresses & status values to the header powerpc/powernv/pmem: Read the capability registers & wait for device ready powerpc/powernv/pmem: Add support for Admin commands powerpc/powernv/pmem: Add support for near storage commands powerpc/powernv/pmem: Register a character device for userspace to interact with powerpc/powernv/pmem: Implement the Read Error Log command powerpc/powernv/pmem: Add controller dump IOCTLs powerpc/powernv/pmem: Add an IOCTL to report controller statistics powerpc/powernv/pmem: Forward events to userspace powerpc/powernv/pmem: Add an IOCTL to request controller health & perf data powerpc/powernv/pmem: Implement the heartbeat command powerpc/powernv/pmem: Add debug IOCTLs powerpc/powernv/pmem: Expose SMART data via ndctl powerpc/powernv/pmem: Expose the serial number in sysfs powerpc/powernv/pmem: Expose the firmware version in sysfs MAINTAINERS: Add myself & nvdimm/ocxl to ocxl MAINTAINERS |3 + arch/powerpc/configs/powernv_defconfig|5 + arch/powerpc/include/asm/opal-api.h |2 + arch/powerpc/include/asm/opal.h |3 + arch/powerpc/include/asm/pnv-ocxl.h | 40 +- arch/powerpc/platforms/powernv/Kconfig|3 + arch/powerpc/platforms/powernv/Makefile |1 + arch/powerpc/platforms/powernv/ocxl.c | 43 + arch/powerpc/platforms/powernv/opal-call.c|2 + arch/powerpc/platforms/powernv/pmem/Kconfig | 21 + arch/powerpc/platforms/powernv/pmem/Makefile |7 + arch/powerpc/platforms/powernv/pmem/ocxl.c| 1991 + .../platforms/powernv/pmem/ocxl_internal.c| 213 ++ .../platforms/powernv/pmem/ocxl_internal.h| 254 +++ .../platforms/powernv/pmem/ocxl_sysfs.c | 46 + drivers/misc/ocxl/config.c| 74 +- drivers/misc/ocxl/core.c | 61 + drivers/misc/ocxl/link.c | 53 + drivers/misc/ocxl/ocxl_internal.h | 45 +- include/linux/memory_hotplug.h|5 + include/misc/ocxl.h | 122 +- include/uapi/linux/ndctl.h|1 + include/uapi/nvdimm/ocxl-pmem.h | 127 ++ mm/memory_hotplug.c |4 +- 24 files changed, 3029 insertions(+), 97 deletions(-) create mode 100644 arch/powerpc/platforms/powernv/pmem/Kconfig create mode 100644 arch/powerpc/platforms/powernv/pmem/Makefile create mode 100644 arch/powerpc/platforms/powernv/pmem/ocxl.c create mode 100644 arch/powerpc/platforms/powernv/pmem/ocxl_internal.c create mode 100644 arch/powerpc/platforms/powernv/pmem/ocxl_internal.h create mode 100644 arch/powerpc/platforms/powernv/pmem/ocxl_sysfs.c create mode 100644 include/uapi/nvdimm/ocxl-pmem.h -- 2.24.1 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 09/27] ocxl: Save the device serial number in ocxl_fn
From: Alastair D'Silva
This patch retrieves the serial number of the card and makes it available
to consumers of the ocxl driver via the ocxl_fn struct.
Signed-off-by: Alastair D'Silva
Acked-by: Frederic Barrat
Acked-by: Andrew Donnellan
---
drivers/misc/ocxl/config.c | 46 ++
include/misc/ocxl.h| 1 +
2 files changed, 47 insertions(+)
diff --git a/drivers/misc/ocxl/config.c b/drivers/misc/ocxl/config.c
index 701ae6216abf..ce33fafa7b50 100644
--- a/drivers/misc/ocxl/config.c
+++ b/drivers/misc/ocxl/config.c
@@ -71,6 +71,51 @@ static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8
afu_idx)
return 0;
}
+/**
+ * get_function_0() - Find a related PCI device (function 0)
+ * @device: PCI device to match
+ *
+ * Returns a pointer to the related device, or null if not found
+ */
+static struct pci_dev *get_function_0(struct pci_dev *dev)
+{
+ unsigned int devfn = PCI_DEVFN(PCI_SLOT(dev->devfn), 0);
+
+ return pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),
+ dev->bus->number, devfn);
+}
+
+static void read_serial(struct pci_dev *dev, struct ocxl_fn_config *fn)
+{
+ u32 low, high;
+ int pos;
+
+ pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN);
+ if (pos) {
+ pci_read_config_dword(dev, pos + 0x04, &low);
+ pci_read_config_dword(dev, pos + 0x08, &high);
+
+ fn->serial = low | ((u64)high) << 32;
+
+ return;
+ }
+
+ if (PCI_FUNC(dev->devfn) != 0) {
+ struct pci_dev *related = get_function_0(dev);
+
+ if (!related) {
+ fn->serial = 0;
+ return;
+ }
+
+ read_serial(related, fn);
+ pci_dev_put(related);
+ return;
+ }
+
+ fn->serial = 0;
+}
+
static void read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn)
{
u16 val;
@@ -208,6 +253,7 @@ int ocxl_config_read_function(struct pci_dev *dev, struct
ocxl_fn_config *fn)
int rc;
read_pasid(dev, fn);
+ read_serial(dev, fn);
rc = read_dvsec_tl(dev, fn);
if (rc) {
diff --git a/include/misc/ocxl.h b/include/misc/ocxl.h
index d8b0b4d46bfb..b8514dc64bd0 100644
--- a/include/misc/ocxl.h
+++ b/include/misc/ocxl.h
@@ -46,6 +46,7 @@ struct ocxl_fn_config {
int dvsec_afu_info_pos; /* offset of the AFU information DVSEC */
s8 max_pasid_log;
s8 max_afu_index;
+ u64 serial;
};
enum ocxl_endian {
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 26/27] powerpc/powernv/pmem: Expose the firmware version in sysfs
From: Alastair D'Silva
This information will be used by ndctl in userspace to help users identify
the device.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/ocxl_sysfs.c | 9 +
1 file changed, 9 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl_sysfs.c
b/arch/powerpc/platforms/powernv/pmem/ocxl_sysfs.c
index 7829e4bc887d..84b23cc3e8b7 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl_sysfs.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl_sysfs.c
@@ -16,8 +16,17 @@ static ssize_t serial_show(struct device *device, struct
device_attribute *attr,
return scnprintf(buf, PAGE_SIZE, "%llu\n", fn_config->serial);
}
+static ssize_t fw_version_show(struct device *device,
+ struct device_attribute *attr, char *buf)
+{
+ struct ocxlpmem *ocxlpmem = container_of(device, struct ocxlpmem, dev);
+
+ return scnprintf(buf, PAGE_SIZE, "%s\n", ocxlpmem->fw_version);
+}
+
static struct device_attribute attrs[] = {
__ATTR_RO(serial),
+ __ATTR_RO(fw_version),
};
int ocxlpmem_sysfs_add(struct ocxlpmem *ocxlpmem)
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 02/27] mm/memory_hotplug: Allow check_hotplug_memory_addressable to be called from drivers
From: Alastair D'Silva
When setting up OpenCAPI connected persistent memory, the range check may
not be performed until quite late (or perhaps not at all, if the user does
not establish a DAX device).
This patch makes the range check callable so we can perform the check while
probing the OpenCAPI SCM device.
Signed-off-by: Alastair D'Silva
---
include/linux/memory_hotplug.h | 5 +
mm/memory_hotplug.c| 4 ++--
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
index f4d59155f3d4..34a69aecc45e 100644
--- a/include/linux/memory_hotplug.h
+++ b/include/linux/memory_hotplug.h
@@ -337,6 +337,11 @@ static inline void __remove_memory(int nid, u64 start, u64
size) {}
extern void set_zone_contiguous(struct zone *zone);
extern void clear_zone_contiguous(struct zone *zone);
+#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
+int check_hotplug_memory_addressable(unsigned long pfn,
+ unsigned long nr_pages);
+#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
+
extern void __ref free_area_init_core_hotplug(int nid);
extern int __add_memory(int nid, u64 start, u64 size);
extern int add_memory(int nid, u64 start, u64 size);
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 0a54ffac8c68..14945f033594 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -276,8 +276,8 @@ static int check_pfn_span(unsigned long pfn, unsigned long
nr_pages,
return 0;
}
-static int check_hotplug_memory_addressable(unsigned long pfn,
- unsigned long nr_pages)
+int check_hotplug_memory_addressable(unsigned long pfn,
+unsigned long nr_pages)
{
const u64 max_addr = PFN_PHYS(pfn + nr_pages) - 1;
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 08/27] ocxl: Emit a log message showing how much LPC memory was detected
From: Alastair D'Silva This patch emits a message showing how much LPC memory & special purpose memory was detected on an OCXL device. Signed-off-by: Alastair D'Silva --- drivers/misc/ocxl/config.c | 4 1 file changed, 4 insertions(+) diff --git a/drivers/misc/ocxl/config.c b/drivers/misc/ocxl/config.c index a62e3d7db2bf..701ae6216abf 100644 --- a/drivers/misc/ocxl/config.c +++ b/drivers/misc/ocxl/config.c @@ -568,6 +568,10 @@ static int read_afu_lpc_memory_info(struct pci_dev *dev, afu->special_purpose_mem_size = total_mem_size - lpc_mem_size; } + + dev_info(&dev->dev, "Probed LPC memory of %#llx bytes and special purpose memory of %#llx bytes\n", + afu->lpc_mem_size, afu->special_purpose_mem_size); + return 0; } -- 2.24.1 ___ Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
[PATCH v3 14/27] powerpc/powernv/pmem: Add support for Admin commands
From: Alastair D'Silva
This patch requests the metadata required to issue admin commands, as well
as some helper functions to construct and check the completion of the
commands.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/ocxl.c| 65
.../platforms/powernv/pmem/ocxl_internal.c| 153 ++
.../platforms/powernv/pmem/ocxl_internal.h| 61 +++
3 files changed, 279 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index 431212c9f0cc..4e782d22605b 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -216,6 +216,58 @@ static int register_lpc_mem(struct ocxlpmem *ocxlpmem)
return 0;
}
+/**
+ * extract_command_metadata() - Extract command data from MMIO & save it for
further use
+ * @ocxlpmem: the device metadata
+ * @offset: The base address of the command data structures (address of CREQO)
+ * @command_metadata: A pointer to the command metadata to populate
+ * Return: 0 on success, negative on failure
+ */
+static int extract_command_metadata(struct ocxlpmem *ocxlpmem, u32 offset,
+ struct command_metadata
*command_metadata)
+{
+ int rc;
+ u64 tmp;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, offset,
OCXL_LITTLE_ENDIAN,
+&tmp);
+ if (rc)
+ return rc;
+
+ command_metadata->request_offset = tmp >> 32;
+ command_metadata->response_offset = tmp & 0x;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, offset + 8,
OCXL_LITTLE_ENDIAN,
+&tmp);
+ if (rc)
+ return rc;
+
+ command_metadata->data_offset = tmp >> 32;
+ command_metadata->data_size = tmp & 0x;
+
+ command_metadata->id = 0;
+
+ return 0;
+}
+
+/**
+ * setup_command_metadata() - Set up the command metadata
+ * @ocxlpmem: the device metadata
+ */
+static int setup_command_metadata(struct ocxlpmem *ocxlpmem)
+{
+ int rc;
+
+ mutex_init(&ocxlpmem->admin_command.lock);
+
+ rc = extract_command_metadata(ocxlpmem, GLOBAL_MMIO_ACMA_CREQO,
+ &ocxlpmem->admin_command);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+
/**
* is_usable() - Is a controller usable?
* @ocxlpmem: the device metadata
@@ -456,6 +508,14 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
}
ocxlpmem->pdev = pdev;
+ ocxlpmem->timeouts[ADMIN_COMMAND_ERRLOG] = 2000; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_HEARTBEAT] = 100; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_SMART] = 100; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_CONTROLLER_DUMP] = 1000; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_CONTROLLER_STATS] = 100; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_SHUTDOWN] = 1000; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_FW_UPDATE] = 16000; // ms
+
pci_set_drvdata(pdev, ocxlpmem);
ocxlpmem->ocxl_fn = ocxl_function_open(pdev);
@@ -501,6 +561,11 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
goto err;
}
+ if (setup_command_metadata(ocxlpmem)) {
+ dev_err(&pdev->dev, "Could not read OCXL command matada\n");
+ goto err;
+ }
+
elapsed = 0;
timeout = ocxlpmem->readiness_timeout +
ocxlpmem->memory_available_timeout;
while (!is_usable(ocxlpmem, false)) {
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
index 617ca943b1b8..583f48023025 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
@@ -17,3 +17,156 @@ int ocxlpmem_chi(const struct ocxlpmem *ocxlpmem, u64 *chi)
return 0;
}
+
+#define COMMAND_REQUEST_SIZE (8 * sizeof(u64))
+static int scm_command_request(const struct ocxlpmem *ocxlpmem,
+ struct command_metadata *cmd, u8 op_code)
+{
+ u64 val = op_code;
+ int rc;
+ u8 i;
+
+ cmd->op_code = op_code;
+ cmd->id++;
+
+ val |= ((u64)cmd->id) << 16;
+
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu, cmd->request_offset,
+ OCXL_LITTLE_ENDIAN, val);
+ if (rc)
+ return rc;
+
+ for (i = sizeof(u64); i < COMMAND_REQUEST_SIZE; i += sizeof(u64)) {
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+ cmd->request_offset + i,
+ OCXL_LITTLE_ENDIAN, 0);
+
[PATCH v3 07/27] ocxl: Add functions to map/unmap LPC memory
From: Alastair D'Silva
Add functions to map/unmap LPC memory
Signed-off-by: Alastair D'Silva
---
drivers/misc/ocxl/core.c | 51 +++
drivers/misc/ocxl/ocxl_internal.h | 3 ++
include/misc/ocxl.h | 21 +
3 files changed, 75 insertions(+)
diff --git a/drivers/misc/ocxl/core.c b/drivers/misc/ocxl/core.c
index 2531c6cf19a0..75ff14e3882a 100644
--- a/drivers/misc/ocxl/core.c
+++ b/drivers/misc/ocxl/core.c
@@ -210,6 +210,56 @@ static void unmap_mmio_areas(struct ocxl_afu *afu)
release_fn_bar(afu->fn, afu->config.global_mmio_bar);
}
+int ocxl_afu_map_lpc_mem(struct ocxl_afu *afu)
+{
+ struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
+
+ if ((afu->config.lpc_mem_size + afu->config.special_purpose_mem_size)
== 0)
+ return 0;
+
+ afu->lpc_base_addr = ocxl_link_lpc_map(afu->fn->link, dev);
+ if (afu->lpc_base_addr == 0)
+ return -EINVAL;
+
+ if (afu->config.lpc_mem_size > 0) {
+ afu->lpc_res.start = afu->lpc_base_addr +
afu->config.lpc_mem_offset;
+ afu->lpc_res.end = afu->lpc_res.start +
afu->config.lpc_mem_size - 1;
+ }
+
+ if (afu->config.special_purpose_mem_size > 0) {
+ afu->special_purpose_res.start = afu->lpc_base_addr +
+
afu->config.special_purpose_mem_offset;
+ afu->special_purpose_res.end = afu->special_purpose_res.start +
+
afu->config.special_purpose_mem_size - 1;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(ocxl_afu_map_lpc_mem);
+
+struct resource *ocxl_afu_lpc_mem(struct ocxl_afu *afu)
+{
+ return &afu->lpc_res;
+}
+EXPORT_SYMBOL_GPL(ocxl_afu_lpc_mem);
+
+static void unmap_lpc_mem(struct ocxl_afu *afu)
+{
+ struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
+
+ if (afu->lpc_res.start || afu->special_purpose_res.start) {
+ void *link = afu->fn->link;
+
+ // only release the link when the the last consumer calls
release
+ ocxl_link_lpc_release(link, dev);
+
+ afu->lpc_res.start = 0;
+ afu->lpc_res.end = 0;
+ afu->special_purpose_res.start = 0;
+ afu->special_purpose_res.end = 0;
+ }
+}
+
static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev)
{
int rc;
@@ -251,6 +301,7 @@ static int configure_afu(struct ocxl_afu *afu, u8 afu_idx,
struct pci_dev *dev)
static void deconfigure_afu(struct ocxl_afu *afu)
{
+ unmap_lpc_mem(afu);
unmap_mmio_areas(afu);
reclaim_afu_pasid(afu);
reclaim_afu_actag(afu);
diff --git a/drivers/misc/ocxl/ocxl_internal.h
b/drivers/misc/ocxl/ocxl_internal.h
index d0c8c4838f42..ce0cac1da416 100644
--- a/drivers/misc/ocxl/ocxl_internal.h
+++ b/drivers/misc/ocxl/ocxl_internal.h
@@ -52,6 +52,9 @@ struct ocxl_afu {
void __iomem *global_mmio_ptr;
u64 pp_mmio_start;
void *private;
+ u64 lpc_base_addr; /* Covers both LPC & special purpose memory */
+ struct resource lpc_res;
+ struct resource special_purpose_res;
};
enum ocxl_context_status {
diff --git a/include/misc/ocxl.h b/include/misc/ocxl.h
index 357ef1aadbc0..d8b0b4d46bfb 100644
--- a/include/misc/ocxl.h
+++ b/include/misc/ocxl.h
@@ -203,6 +203,27 @@ int ocxl_irq_set_handler(struct ocxl_context *ctx, int
irq_id,
// AFU Metadata
+/**
+ * ocxl_afu_map_lpc_mem() - Map the LPC system & special purpose memory for an
AFU
+ * Do not call this during device discovery, as there may me multiple
+ * devices on a link, and the memory is mapped for the whole link, not
+ * just one device. It should only be called after all devices have
+ * registered their memory on the link.
+ *
+ * @afu: The AFU that has the LPC memory to map
+ *
+ * Returns 0 on success, negative on failure
+ */
+int ocxl_afu_map_lpc_mem(struct ocxl_afu *afu);
+
+/**
+ * ocxl_afu_lpc_mem() - Get the physical address range of LPC memory for an AFU
+ * @afu: The AFU associated with the LPC memory
+ *
+ * Returns a pointer to the resource struct for the physical address range
+ */
+struct resource *ocxl_afu_lpc_mem(struct ocxl_afu *afu);
+
/**
* ocxl_afu_config() - Get a pointer to the config for an AFU
* @afu: a pointer to the AFU to get the config for
--
2.24.1
___
Linux-nvdimm mailing list -- linux-nvdimm@lists.01.org
To unsubscribe send an email to linux-nvdimm-le...@lists.01.org
RE: [PATCH v2 24/27] nvdimm/ocxl: Implement Overwrite
On Mon, 2020-02-03 at 15:10 +, Jonathan Cameron wrote:
> On Tue, 3 Dec 2019 14:46:52 +1100
> Alastair D'Silva wrote:
>
> > From: Alastair D'Silva
> >
> > The near storage command 'Secure Erase' overwrites all data on the
> > media.
> >
> > This patch hooks it up to the security function 'overwrite'.
> >
> > Signed-off-by: Alastair D'Silva
>
> A few things to tidy up in here.
>
> Thanks,
>
> Jonathan
>
>
> > ---
> > drivers/nvdimm/ocxl/scm.c | 164
> > -
> > drivers/nvdimm/ocxl/scm_internal.c | 1 +
> > drivers/nvdimm/ocxl/scm_internal.h | 17 +++
> > 3 files changed, 180 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/nvdimm/ocxl/scm.c b/drivers/nvdimm/ocxl/scm.c
> > index a81eb5916eb3..8deb7862793c 100644
> > --- a/drivers/nvdimm/ocxl/scm.c
> > +++ b/drivers/nvdimm/ocxl/scm.c
> > @@ -169,6 +169,86 @@ static int scm_reserve_metadata(struct
> > scm_data *scm_data,
> > return 0;
> > }
> >
> > +/**
> > + * scm_overwrite() - Overwrite all data on the card
> > + * @scm_data: The SCM device data
>
> I would mention in here that this exists with the lock held and
> where that is unlocked again.
Ok
>
> > + * Return: 0 on success
> > + */
> > +int scm_overwrite(struct scm_data *scm_data)
> > +{
> > + int rc;
> > +
> > + mutex_lock(&scm_data->ns_command.lock);
> > +
> > + rc = scm_ns_command_request(scm_data, NS_COMMAND_SECURE_ERASE);
> > + if (rc)
>
> Perhaps change that goto label to reflect it is the error path rather
> than a shared exit route.
>
Ok
> > + goto out;
> > +
> > + rc = scm_ns_command_execute(scm_data);
> > + if (rc)
> > + goto out;
> > +
> > + scm_data->overwrite_state = SCM_OVERWRITE_BUSY;
> > +
> > + return 0;
> > +
> > +out:
> > + mutex_unlock(&scm_data->ns_command.lock);
> > + return rc;
> > +}
> > +
> > +/**
> > + * scm_secop_overwrite() - Overwrite all data on the card
> > + * @nvdimm: The nvdimm representation of the SCM device to start
> > the overwrite on
> > + * @key_data: Unused (no security key implementation)
> > + * Return: 0 on success
> > + */
> > +static int scm_secop_overwrite(struct nvdimm *nvdimm,
> > + const struct nvdimm_key_data *key_data)
> > +{
> > + struct scm_data *scm_data = nvdimm_provider_data(nvdimm);
> > +
> > + return scm_overwrite(scm_data);
> > +}
> > +
> > +/**
> > + * scm_secop_query_overwrite() - Get the current overwrite state
> > + * @nvdimm: The nvdimm representation of the SCM device to start
> > the overwrite on
> > + * Return: 0 if successful or idle, -EBUSY if busy, -EFAULT if
> > failed
> > + */
> > +static int scm_secop_query_overwrite(struct nvdimm *nvdimm)
> > +{
> > + struct scm_data *scm_data = nvdimm_provider_data(nvdimm);
> > +
> > + if (scm_data->overwrite_state == SCM_OVERWRITE_BUSY)
> > + return -EBUSY;
> > +
> > + if (scm_data->overwrite_state == SCM_OVERWRITE_FAILED)
> > + return -EFAULT;
> > +
> > + return 0;
> > +}
> > +
> > +/**
> > + * scm_secop_get_flags() - return the security flags for the SCM
> > device
>
> All params need to documented in kernel-doc comments.
Ok
>
> > + */
> > +static unsigned long scm_secop_get_flags(struct nvdimm *nvdimm,
> > + enum nvdimm_passphrase_type ptype)
> > +{
> > + struct scm_data *scm_data = nvdimm_provider_data(nvdimm);
> > +
> > + if (scm_data->overwrite_state == SCM_OVERWRITE_BUSY)
> > + return BIT(NVDIMM_SECURITY_OVERWRITE);
> > +
> > + return BIT(NVDIMM_SECURITY_DISABLED);
> > +}
> > +
> > +static const struct nvdimm_security_ops sec_ops = {
> > + .get_flags = scm_secop_get_flags,
> > + .overwrite = scm_secop_overwrite,
> > + .query_overwrite = scm_secop_query_overwrite,
> > +};
> > +
> > /**
> > * scm_register_lpc_mem() - Discover persistent memory on a device
> > and register it with the NVDIMM subsystem
> > * @scm_data: The SCM device data
> > @@ -224,10 +304,10 @@ static int scm_register_lpc_mem(struct
> > scm_data *scm_data)
> > set_bit(NDD_ALIASING, &nvdimm_flags);
> >
> > snprintf(serial, sizeof(serial), "%llx", fn_config->serial);
&
