RE: [PATCH v3 14/27] powerpc/powernv/pmem: Add support for Admin commands
On Thu, 2020-02-27 at 09:01 -0800, Dan Williams wrote: > On Thu, Feb 20, 2020 at 7:28 PM Alastair D'Silva < > alast...@au1.ibm.com> wrote: > > From: Alastair D'Silva > > > > This patch requests the metadata required to issue admin commands, > > as well > > as some helper functions to construct and check the completion of > > the > > commands. > > What are the admin commands? Any pointer to a spec? Why does Linux > need to support these commands? I'll flesh these out for the next spin, thanks. -- Alastair D'Silva Open Source Developer Linux Technology Centre, IBM Australia mob: 0423 762 819
Re: [PATCH v3 14/27] powerpc/powernv/pmem: Add support for Admin commands
On Thu, 2020-02-27 at 19:27 +1100, Andrew Donnellan wrote:
> On 27/2/20 7:22 pm, Andrew Donnellan wrote:
> > > +int admin_command_request(struct ocxlpmem *ocxlpmem, u8 op_code)
> > > +{
> > > +u64 val;
> > > +int rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
> > > GLOBAL_MMIO_CHI,
> > > + OCXL_LITTLE_ENDIAN, );
> > > +if (rc)
> > > +return rc;
> >
> > Ignoring the value here expected, you're just trying to verify that
> > you
> > don't see an error on the read?
>
> I see that in the next patch, in ns_command_request() you check that
> NSCRA is 1 - did you mean to check that ACRA = 1 here?
>
>
I was in one version, but that was causing problems in startup since
there was successful prior command to assert ACRA.
I should remove the NSCRA check too.
--
Alastair D'Silva
Open Source Developer
Linux Technology Centre, IBM Australia
mob: 0423 762 819
Re: [PATCH v3 14/27] powerpc/powernv/pmem: Add support for Admin commands
On Thu, 2020-02-27 at 19:22 +1100, Andrew Donnellan wrote:
> On 21/2/20 2:27 pm, Alastair D'Silva wrote:
> > From: Alastair D'Silva
> >
> > This patch requests the metadata required to issue admin commands,
> > as well
> > as some helper functions to construct and check the completion of
> > the
> > commands.
> >
> > Signed-off-by: Alastair D'Silva
> > ---
> > arch/powerpc/platforms/powernv/pmem/ocxl.c| 65
> > .../platforms/powernv/pmem/ocxl_internal.c| 153
> > ++
> > .../platforms/powernv/pmem/ocxl_internal.h| 61 +++
> > 3 files changed, 279 insertions(+)
> >
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > index 431212c9f0cc..4e782d22605b 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
> > @@ -216,6 +216,58 @@ static int register_lpc_mem(struct ocxlpmem
> > *ocxlpmem)
> > return 0;
> > }
> >
> > +/**
> > + * extract_command_metadata() - Extract command data from MMIO &
> > save it for further use
> > + * @ocxlpmem: the device metadata
> > + * @offset: The base address of the command data structures
> > (address of CREQO)
> > + * @command_metadata: A pointer to the command metadata to
> > populate
> > + * Return: 0 on success, negative on failure
> > + */
> > +static int extract_command_metadata(struct ocxlpmem *ocxlpmem, u32
> > offset,
> > + struct command_metadata
> > *command_metadata)
> > +{
> > + int rc;
> > + u64 tmp;
> > +
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, offset,
> > OCXL_LITTLE_ENDIAN,
> > +);
> > + if (rc)
> > + return rc;
> > +
> > + command_metadata->request_offset = tmp >> 32;
> > + command_metadata->response_offset = tmp & 0x;
> > +
> > + rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, offset + 8,
> > OCXL_LITTLE_ENDIAN,
> > +);
> > + if (rc)
> > + return rc;
> > +
> > + command_metadata->data_offset = tmp >> 32;
> > + command_metadata->data_size = tmp & 0x;
> > +
> > + command_metadata->id = 0;
> > +
> > + return 0;
> > +}
> > +
> > +/**
> > + * setup_command_metadata() - Set up the command metadata
> > + * @ocxlpmem: the device metadata
> > + */
> > +static int setup_command_metadata(struct ocxlpmem *ocxlpmem)
> > +{
> > + int rc;
> > +
> > + mutex_init(>admin_command.lock);
> > +
> > + rc = extract_command_metadata(ocxlpmem, GLOBAL_MMIO_ACMA_CREQO,
> > + >admin_command);
> > + if (rc)
> > + return rc;
> > +
> > + return 0;
> > +}
> > +
> > /**
> >* is_usable() - Is a controller usable?
> >* @ocxlpmem: the device metadata
> > @@ -456,6 +508,14 @@ static int probe(struct pci_dev *pdev, const
> > struct pci_device_id *ent)
> > }
> > ocxlpmem->pdev = pdev;
> >
> > + ocxlpmem->timeouts[ADMIN_COMMAND_ERRLOG] = 2000; // ms
> > + ocxlpmem->timeouts[ADMIN_COMMAND_HEARTBEAT] = 100; // ms
> > + ocxlpmem->timeouts[ADMIN_COMMAND_SMART] = 100; // ms
> > + ocxlpmem->timeouts[ADMIN_COMMAND_CONTROLLER_DUMP] = 1000; // ms
> > + ocxlpmem->timeouts[ADMIN_COMMAND_CONTROLLER_STATS] = 100; // ms
> > + ocxlpmem->timeouts[ADMIN_COMMAND_SHUTDOWN] = 1000; // ms
> > + ocxlpmem->timeouts[ADMIN_COMMAND_FW_UPDATE] = 16000; // ms
>
> Why are we keeping these timeouts in a per device struct? I can't
> see
> anywhere where we change these values.
>
These are overwritten in a later patch, which I've missed! thanks for
pointing this out.
These initial values will be overwritten by card specific timeouts.
> > +
> > pci_set_drvdata(pdev, ocxlpmem);
> >
> > ocxlpmem->ocxl_fn = ocxl_function_open(pdev);
> > @@ -501,6 +561,11 @@ static int probe(struct pci_dev *pdev, const
> > struct pci_device_id *ent)
> > goto err;
> > }
> >
> > + if (setup_command_metadata(ocxlpmem)) {
> > + dev_err(>dev, "Could not read OCXL command
> > matada\n");
>
> metadata
Wow, not sure how that happened.
>
> Also, "OCXL command metadata" is misleading, this is a pmem specific
> thing, not an OpenCAPI thing, I would prefer just "command metadata".
>
Ok
> > + goto err;
> > + }
> > +
> > elapsed = 0;
> > timeout = ocxlpmem->readiness_timeout + ocxlpmem-
> > >memory_available_timeout;
> > while (!is_usable(ocxlpmem, false)) {
> > diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
> > b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
> > index 617ca943b1b8..583f48023025 100644
> > --- a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
> > +++ b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
> > @@ -17,3 +17,156 @@ int ocxlpmem_chi(const struct ocxlpmem
> > *ocxlpmem, u64 *chi)
> >
> > return 0;
> > }
> > +
> > +#define COMMAND_REQUEST_SIZE (8 * sizeof(u64))
> > +static int
Re: [PATCH v3 14/27] powerpc/powernv/pmem: Add support for Admin commands
On Thu, Feb 20, 2020 at 7:28 PM Alastair D'Silva wrote: > > From: Alastair D'Silva > > This patch requests the metadata required to issue admin commands, as well > as some helper functions to construct and check the completion of the > commands. What are the admin commands? Any pointer to a spec? Why does Linux need to support these commands?
Re: [PATCH v3 14/27] powerpc/powernv/pmem: Add support for Admin commands
On 27/2/20 7:22 pm, Andrew Donnellan wrote:
+int admin_command_request(struct ocxlpmem *ocxlpmem, u8 op_code)
+{
+ u64 val;
+ int rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu,
GLOBAL_MMIO_CHI,
+ OCXL_LITTLE_ENDIAN, );
+ if (rc)
+ return rc;
Ignoring the value here expected, you're just trying to verify that you
don't see an error on the read?
I see that in the next patch, in ns_command_request() you check that
NSCRA is 1 - did you mean to check that ACRA = 1 here?
--
Andrew Donnellan OzLabs, ADL Canberra
a...@linux.ibm.com IBM Australia Limited
Re: [PATCH v3 14/27] powerpc/powernv/pmem: Add support for Admin commands
On 21/2/20 2:27 pm, Alastair D'Silva wrote:
From: Alastair D'Silva
This patch requests the metadata required to issue admin commands, as well
as some helper functions to construct and check the completion of the
commands.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/ocxl.c| 65
.../platforms/powernv/pmem/ocxl_internal.c| 153 ++
.../platforms/powernv/pmem/ocxl_internal.h| 61 +++
3 files changed, 279 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index 431212c9f0cc..4e782d22605b 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -216,6 +216,58 @@ static int register_lpc_mem(struct ocxlpmem *ocxlpmem)
return 0;
}
+/**
+ * extract_command_metadata() - Extract command data from MMIO & save it for
further use
+ * @ocxlpmem: the device metadata
+ * @offset: The base address of the command data structures (address of CREQO)
+ * @command_metadata: A pointer to the command metadata to populate
+ * Return: 0 on success, negative on failure
+ */
+static int extract_command_metadata(struct ocxlpmem *ocxlpmem, u32 offset,
+ struct command_metadata
*command_metadata)
+{
+ int rc;
+ u64 tmp;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, offset,
OCXL_LITTLE_ENDIAN,
+);
+ if (rc)
+ return rc;
+
+ command_metadata->request_offset = tmp >> 32;
+ command_metadata->response_offset = tmp & 0x;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, offset + 8,
OCXL_LITTLE_ENDIAN,
+);
+ if (rc)
+ return rc;
+
+ command_metadata->data_offset = tmp >> 32;
+ command_metadata->data_size = tmp & 0x;
+
+ command_metadata->id = 0;
+
+ return 0;
+}
+
+/**
+ * setup_command_metadata() - Set up the command metadata
+ * @ocxlpmem: the device metadata
+ */
+static int setup_command_metadata(struct ocxlpmem *ocxlpmem)
+{
+ int rc;
+
+ mutex_init(>admin_command.lock);
+
+ rc = extract_command_metadata(ocxlpmem, GLOBAL_MMIO_ACMA_CREQO,
+ >admin_command);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+
/**
* is_usable() - Is a controller usable?
* @ocxlpmem: the device metadata
@@ -456,6 +508,14 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
}
ocxlpmem->pdev = pdev;
+ ocxlpmem->timeouts[ADMIN_COMMAND_ERRLOG] = 2000; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_HEARTBEAT] = 100; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_SMART] = 100; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_CONTROLLER_DUMP] = 1000; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_CONTROLLER_STATS] = 100; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_SHUTDOWN] = 1000; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_FW_UPDATE] = 16000; // ms
Why are we keeping these timeouts in a per device struct? I can't see
anywhere where we change these values.
+
pci_set_drvdata(pdev, ocxlpmem);
ocxlpmem->ocxl_fn = ocxl_function_open(pdev);
@@ -501,6 +561,11 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
goto err;
}
+ if (setup_command_metadata(ocxlpmem)) {
+ dev_err(>dev, "Could not read OCXL command matada\n");
metadata
Also, "OCXL command metadata" is misleading, this is a pmem specific
thing, not an OpenCAPI thing, I would prefer just "command metadata".
+ goto err;
+ }
+
elapsed = 0;
timeout = ocxlpmem->readiness_timeout +
ocxlpmem->memory_available_timeout;
while (!is_usable(ocxlpmem, false)) {
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
index 617ca943b1b8..583f48023025 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
@@ -17,3 +17,156 @@ int ocxlpmem_chi(const struct ocxlpmem *ocxlpmem, u64 *chi)
return 0;
}
+
+#define COMMAND_REQUEST_SIZE (8 * sizeof(u64))
+static int scm_command_request(const struct ocxlpmem *ocxlpmem,
+ struct command_metadata *cmd, u8 op_code)
+{
+ u64 val = op_code;
+ int rc;
+ u8 i;
+
+ cmd->op_code = op_code;
+ cmd->id++;
+
+ val |= ((u64)cmd->id) << 16;
+
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu, cmd->request_offset,
+ OCXL_LITTLE_ENDIAN, val);
+ if (rc)
+ return rc;
+
+ for (i = sizeof(u64); i < COMMAND_REQUEST_SIZE; i += sizeof(u64)) {
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+
[PATCH v3 14/27] powerpc/powernv/pmem: Add support for Admin commands
From: Alastair D'Silva
This patch requests the metadata required to issue admin commands, as well
as some helper functions to construct and check the completion of the
commands.
Signed-off-by: Alastair D'Silva
---
arch/powerpc/platforms/powernv/pmem/ocxl.c| 65
.../platforms/powernv/pmem/ocxl_internal.c| 153 ++
.../platforms/powernv/pmem/ocxl_internal.h| 61 +++
3 files changed, 279 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl.c
b/arch/powerpc/platforms/powernv/pmem/ocxl.c
index 431212c9f0cc..4e782d22605b 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl.c
@@ -216,6 +216,58 @@ static int register_lpc_mem(struct ocxlpmem *ocxlpmem)
return 0;
}
+/**
+ * extract_command_metadata() - Extract command data from MMIO & save it for
further use
+ * @ocxlpmem: the device metadata
+ * @offset: The base address of the command data structures (address of CREQO)
+ * @command_metadata: A pointer to the command metadata to populate
+ * Return: 0 on success, negative on failure
+ */
+static int extract_command_metadata(struct ocxlpmem *ocxlpmem, u32 offset,
+ struct command_metadata
*command_metadata)
+{
+ int rc;
+ u64 tmp;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, offset,
OCXL_LITTLE_ENDIAN,
+);
+ if (rc)
+ return rc;
+
+ command_metadata->request_offset = tmp >> 32;
+ command_metadata->response_offset = tmp & 0x;
+
+ rc = ocxl_global_mmio_read64(ocxlpmem->ocxl_afu, offset + 8,
OCXL_LITTLE_ENDIAN,
+);
+ if (rc)
+ return rc;
+
+ command_metadata->data_offset = tmp >> 32;
+ command_metadata->data_size = tmp & 0x;
+
+ command_metadata->id = 0;
+
+ return 0;
+}
+
+/**
+ * setup_command_metadata() - Set up the command metadata
+ * @ocxlpmem: the device metadata
+ */
+static int setup_command_metadata(struct ocxlpmem *ocxlpmem)
+{
+ int rc;
+
+ mutex_init(>admin_command.lock);
+
+ rc = extract_command_metadata(ocxlpmem, GLOBAL_MMIO_ACMA_CREQO,
+ >admin_command);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+
/**
* is_usable() - Is a controller usable?
* @ocxlpmem: the device metadata
@@ -456,6 +508,14 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
}
ocxlpmem->pdev = pdev;
+ ocxlpmem->timeouts[ADMIN_COMMAND_ERRLOG] = 2000; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_HEARTBEAT] = 100; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_SMART] = 100; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_CONTROLLER_DUMP] = 1000; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_CONTROLLER_STATS] = 100; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_SHUTDOWN] = 1000; // ms
+ ocxlpmem->timeouts[ADMIN_COMMAND_FW_UPDATE] = 16000; // ms
+
pci_set_drvdata(pdev, ocxlpmem);
ocxlpmem->ocxl_fn = ocxl_function_open(pdev);
@@ -501,6 +561,11 @@ static int probe(struct pci_dev *pdev, const struct
pci_device_id *ent)
goto err;
}
+ if (setup_command_metadata(ocxlpmem)) {
+ dev_err(>dev, "Could not read OCXL command matada\n");
+ goto err;
+ }
+
elapsed = 0;
timeout = ocxlpmem->readiness_timeout +
ocxlpmem->memory_available_timeout;
while (!is_usable(ocxlpmem, false)) {
diff --git a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
index 617ca943b1b8..583f48023025 100644
--- a/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
+++ b/arch/powerpc/platforms/powernv/pmem/ocxl_internal.c
@@ -17,3 +17,156 @@ int ocxlpmem_chi(const struct ocxlpmem *ocxlpmem, u64 *chi)
return 0;
}
+
+#define COMMAND_REQUEST_SIZE (8 * sizeof(u64))
+static int scm_command_request(const struct ocxlpmem *ocxlpmem,
+ struct command_metadata *cmd, u8 op_code)
+{
+ u64 val = op_code;
+ int rc;
+ u8 i;
+
+ cmd->op_code = op_code;
+ cmd->id++;
+
+ val |= ((u64)cmd->id) << 16;
+
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu, cmd->request_offset,
+ OCXL_LITTLE_ENDIAN, val);
+ if (rc)
+ return rc;
+
+ for (i = sizeof(u64); i < COMMAND_REQUEST_SIZE; i += sizeof(u64)) {
+ rc = ocxl_global_mmio_write64(ocxlpmem->ocxl_afu,
+ cmd->request_offset + i,
+ OCXL_LITTLE_ENDIAN, 0);
+ if (rc)
+ return rc;
+ }
+
+ return 0;
+}
+
+int admin_command_request(struct ocxlpmem *ocxlpmem, u8 op_code)
+{
+ u64 val;
+ int rc =
