Re: [PATCH v7 10/12] powerpc/vas: Define vas_win_close() interface

[Michael Ellerman]

Sukadev Bhattiprolu  writes:

> Michael Ellerman [m...@ellerman.id.au] wrote:
>> Sukadev Bhattiprolu  writes:
>> > diff --git a/arch/powerpc/platforms/powernv/vas-window.c 
>> > b/arch/powerpc/platforms/powernv/vas-window.c
>> > index 2dd4b63..24288dd 100644
>> > --- a/arch/powerpc/platforms/powernv/vas-window.c
>> > +++ b/arch/powerpc/platforms/powernv/vas-window.c
>> > @@ -879,11 +887,92 @@ struct vas_window *vas_rx_win_open(int vasid, enum 
>> > vas_cop_type cop,
>> >  }
>> >  EXPORT_SYMBOL_GPL(vas_rx_win_open);
>> >  
>> > -/* stub for now */
>> > +static void poll_window_busy_state(struct vas_window *window)
>> > +{
>> > +  int busy;
>> > +  uint64_t val;
>> > +
>> > +retry:
>> > +  /*
>> > +   * Poll Window Busy flag
>> > +   */
>> > +  val = read_hvwc_reg(window, VREG(WIN_STATUS));
>> > +  busy = GET_FIELD(VAS_WIN_BUSY, val);
>> > +  if (busy) {
>> > +  val = 0;
>> > +  schedule_timeout(2000);
>> 
>> What's 2000?
>> 
>> That's in jiffies, so it's not a fixed amount of time.
>> 
>> But on a typical config that will be 20 _seconds_ ?!
>
> Ok. Should I change to that just HZ and

Does the hardware give us any idea how long we need to wait?

HZ is better I guess but it's still a long time.

>> But you haven't set the task state, so AFAIK it will just return
>> instantly.
>
> call set_current_state(TASK_UNINTERRUPTIBLE) before the schedule_timeout()?

Yes.

>> > +  val = read_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL));
>> > +  cached = GET_FIELD(VAS_WIN_CACHE_STATUS, val);
>> > +  if (cached) {
>> > +  val = 0ULL;
>> > +  val = SET_FIELD(VAS_CASTOUT_REQ, val, 1);
>> > +  val = SET_FIELD(VAS_PUSH_TO_MEM, val, 0);
>> > +  write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);
>> 
>> Sigh, I still don't like that macro :)
>
> :-) For one thing, I have used it a lot now and secondly isn't it easier
> to know that VAS_CASTOUT_REQ bit is set to 1 without worrying about its
> bit position?

Yeah I guess it depends what you're doing. Are you comparing the code to
the spec, where seeing the name is probably what you want?

Or are you debugging the code running on hardware, where you have a dump
of register values or a trace etc. and you don't have any names just hex
values.

At least in my experience I spend more time doing the latter, so being
able to match the hex values in the code up with values I see in memory
or in registers is preferable.

But it's your code so I'll stop whining about that macro :)


cheers

Re: [PATCH v7 10/12] powerpc/vas: Define vas_win_close() interface

[Sukadev Bhattiprolu]

Michael Ellerman [m...@ellerman.id.au] wrote:
> Hi Suka,
> 
> More comments :)

Thanks!

> 
> Sukadev Bhattiprolu  writes:
> 
> > diff --git a/arch/powerpc/platforms/powernv/vas-window.c 
> > b/arch/powerpc/platforms/powernv/vas-window.c
> > index 2dd4b63..24288dd 100644
> > --- a/arch/powerpc/platforms/powernv/vas-window.c
> > +++ b/arch/powerpc/platforms/powernv/vas-window.c
> > @@ -879,11 +887,92 @@ struct vas_window *vas_rx_win_open(int vasid, enum 
> > vas_cop_type cop,
> >  }
> >  EXPORT_SYMBOL_GPL(vas_rx_win_open);
> >  
> > -/* stub for now */
> > +static void poll_window_busy_state(struct vas_window *window)
> > +{
> > +   int busy;
> > +   uint64_t val;
> > +
> > +retry:
> > +   /*
> > +* Poll Window Busy flag
> > +*/
> > +   val = read_hvwc_reg(window, VREG(WIN_STATUS));
> > +   busy = GET_FIELD(VAS_WIN_BUSY, val);
> > +   if (busy) {
> > +   val = 0;
> > +   schedule_timeout(2000);
> 
> What's 2000?
> 
> That's in jiffies, so it's not a fixed amount of time.
> 
> But on a typical config that will be 20 _seconds_ ?!

Ok. Should I change to that just HZ and

> 
> But you haven't set the task state, so AFAIK it will just return
> instantly.

call set_current_state(TASK_UNINTERRUPTIBLE) before the schedule_timeout()?

> 
> And if there's a software/hardware bug and it never stops being busy,
> then we have a softlockup. The other option would be print a big fat
> warning and just not free the window. But maybe that doesn't work for
> other reasons.
> 
> > +   goto retry;
> > +   }
> > +}
> > +
> > +static void poll_window_castout(struct vas_window *window)
> > +{
> > +   int cached;
> > +   uint64_t val;
> > +
> > +   /* Cast window context out of the cache */
> > +retry:
> > +   val = read_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL));
> > +   cached = GET_FIELD(VAS_WIN_CACHE_STATUS, val);
> > +   if (cached) {
> > +   val = 0ULL;
> > +   val = SET_FIELD(VAS_CASTOUT_REQ, val, 1);
> > +   val = SET_FIELD(VAS_PUSH_TO_MEM, val, 0);
> > +   write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);
> 
> Sigh, I still don't like that macro :)

:-) For one thing, I have used it a lot now and secondly isn't it easier
to know that VAS_CASTOUT_REQ bit is set to 1 without worrying about its
bit position? When debugging, yes we have to ensure VAS_CASTOUT_REQ is
properly defined and we have to work out value in "val".

> 
> or:
>   write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 1ull << 63);
> 
> > +
> > +   schedule_timeout(2000);
> > +   goto retry;
> > +   }
> > +}
> > +
> > +/*
> > + * Close a window.
> > + *
> > + * See Section 1.12.1 of VAS workbook v1.05 for details on closing window:
> > + * - Disable new paste operations (unmap paste address)
> > + * - Poll for the "Window Busy" bit to be cleared
> > + * - Clear the Open/Enable bit for the Window.
> > + * - Poll for return of window Credits (implies FIFO empty for Rx win?)
> > + * - Unpin and cast window context out of cache
> > + *
> > + * Besides the hardware, kernel has some bookkeeping of course.
> > + */
> >  int vas_win_close(struct vas_window *window)
> >  {
> > -   return -1;
> > +   uint64_t val;
> > +
> > +   if (!window)
> > +   return 0;
> > +
> > +   if (!window->tx_win && atomic_read(>num_txwins) != 0) {
> > +   pr_devel("VAS: Attempting to close an active Rx window!\n");
> > +   WARN_ON_ONCE(1);
> > +   return -EAGAIN;
> 
> EAGAIN means "if you do the same thing again it might work".
> 
> I don't think that's right here. The window is not in a state where it
> can be freed, the caller needs to do something to fix that.
> 
> EBUSY would probably be more appropriate.

Ok. Should not happen now (or even with the fast thread-wake up code)
since only the kernel should be closing the windows - so its really a
bug.  Will change to EBUSY though.
> 
> 
> cheers

Re: [PATCH v7 10/12] powerpc/vas: Define vas_win_close() interface

[Michael Ellerman]

Hi Suka,

More comments :)

Sukadev Bhattiprolu  writes:

> diff --git a/arch/powerpc/platforms/powernv/vas-window.c 
> b/arch/powerpc/platforms/powernv/vas-window.c
> index 2dd4b63..24288dd 100644
> --- a/arch/powerpc/platforms/powernv/vas-window.c
> +++ b/arch/powerpc/platforms/powernv/vas-window.c
> @@ -879,11 +887,92 @@ struct vas_window *vas_rx_win_open(int vasid, enum 
> vas_cop_type cop,
>  }
>  EXPORT_SYMBOL_GPL(vas_rx_win_open);
>  
> -/* stub for now */
> +static void poll_window_busy_state(struct vas_window *window)
> +{
> + int busy;
> + uint64_t val;
> +
> +retry:
> + /*
> +  * Poll Window Busy flag
> +  */
> + val = read_hvwc_reg(window, VREG(WIN_STATUS));
> + busy = GET_FIELD(VAS_WIN_BUSY, val);
> + if (busy) {
> + val = 0;
> + schedule_timeout(2000);

What's 2000?

That's in jiffies, so it's not a fixed amount of time.

But on a typical config that will be 20 _seconds_ ?!

But you haven't set the task state, so AFAIK it will just return
instantly.

And if there's a software/hardware bug and it never stops being busy,
then we have a softlockup. The other option would be print a big fat
warning and just not free the window. But maybe that doesn't work for
other reasons.

> + goto retry;
> + }
> +}
> +
> +static void poll_window_castout(struct vas_window *window)
> +{
> + int cached;
> + uint64_t val;
> +
> + /* Cast window context out of the cache */
> +retry:
> + val = read_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL));
> + cached = GET_FIELD(VAS_WIN_CACHE_STATUS, val);
> + if (cached) {
> + val = 0ULL;
> + val = SET_FIELD(VAS_CASTOUT_REQ, val, 1);
> + val = SET_FIELD(VAS_PUSH_TO_MEM, val, 0);
> + write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);

Sigh, I still don't like that macro :)

or:
write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 1ull << 63);

> +
> + schedule_timeout(2000);
> + goto retry;
> + }
> +}
> +
> +/*
> + * Close a window.
> + *
> + * See Section 1.12.1 of VAS workbook v1.05 for details on closing window:
> + *   - Disable new paste operations (unmap paste address)
> + *   - Poll for the "Window Busy" bit to be cleared
> + *   - Clear the Open/Enable bit for the Window.
> + *   - Poll for return of window Credits (implies FIFO empty for Rx win?)
> + *   - Unpin and cast window context out of cache
> + *
> + * Besides the hardware, kernel has some bookkeeping of course.
> + */
>  int vas_win_close(struct vas_window *window)
>  {
> - return -1;
> + uint64_t val;
> +
> + if (!window)
> + return 0;
> +
> + if (!window->tx_win && atomic_read(>num_txwins) != 0) {
> + pr_devel("VAS: Attempting to close an active Rx window!\n");
> + WARN_ON_ONCE(1);
> + return -EAGAIN;

EAGAIN means "if you do the same thing again it might work".

I don't think that's right here. The window is not in a state where it
can be freed, the caller needs to do something to fix that.

EBUSY would probably be more appropriate.


cheers

[PATCH v7 10/12] powerpc/vas: Define vas_win_close() interface

[Sukadev Bhattiprolu]

Define the vas_win_close() interface which should be used to close a
send or receive windows.

While the hardware configurations required to open send and receive windows
differ, the configuration to close a window is the same for both. So we use
a single interface to close the window.

Signed-off-by: Sukadev Bhattiprolu 
---
Changelog[v4]:
- Drop the poll for credits return (we can set the required credit,
  but cannot really find the available credit at a point in time)
- Export the symbol

Changelog[v3]:
- Fix order of parameters in GET_FIELD().
- Update references and sequence for closing/quiescing a window.
---
 arch/powerpc/include/asm/vas.h  |  7 ++
 arch/powerpc/platforms/powernv/vas-window.c | 99 +++--
 2 files changed, 101 insertions(+), 5 deletions(-)

diff --git a/arch/powerpc/include/asm/vas.h b/arch/powerpc/include/asm/vas.h
index 0701a08..3478ac8 100644
--- a/arch/powerpc/include/asm/vas.h
+++ b/arch/powerpc/include/asm/vas.h
@@ -109,4 +109,11 @@ extern void vas_init_rx_win_attr(struct vas_rx_win_attr 
*rxattr,
 extern struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop,
struct vas_rx_win_attr *attr);
 
+/*
+ * Close the send or receive window identified by @win. For receive windows
+ * return -EAGAIN if there are active send windows attached to this receive
+ * window.
+ */
+int vas_win_close(struct vas_window *win);
+
 #endif /* _MISC_VAS_H */
diff --git a/arch/powerpc/platforms/powernv/vas-window.c 
b/arch/powerpc/platforms/powernv/vas-window.c
index 2dd4b63..24288dd 100644
--- a/arch/powerpc/platforms/powernv/vas-window.c
+++ b/arch/powerpc/platforms/powernv/vas-window.c
@@ -139,7 +139,7 @@ static void unmap_region(void *addr, uint64_t start, int 
len)
 /*
  * Unmap the paste address region for a window.
  */
-void unmap_paste_region(struct vas_window *window)
+static void unmap_paste_region(struct vas_window *window)
 {
int len;
uint64_t busaddr_start;
@@ -535,7 +535,7 @@ int vas_assign_window_id(struct ida *ida)
return winid;
 }
 
-void vas_window_free(struct vas_window *window)
+static void vas_window_free(struct vas_window *window)
 {
int winid = window->winid;
struct vas_instance *vinst = window->vinst;
@@ -572,6 +572,14 @@ static struct vas_window *vas_window_alloc(struct 
vas_instance *vinst)
return ERR_PTR(-ENOMEM);
 }
 
+static void put_rx_win(struct vas_window *rxwin)
+{
+   /* Better not be a send window! */
+   WARN_ON_ONCE(rxwin->tx_win);
+
+   atomic_dec(>num_txwins);
+}
+
 /*
  * Find the user space receive window given the @pswid.
  *  - We must have a valid vasid and it must belong to this instance.
@@ -664,7 +672,7 @@ static void set_vinst_win(struct vas_instance *vinst,
  * Clear this window from the table(s) of windows for this VAS instance.
  * See also function header of set_vinst_win().
  */
-void clear_vinst_win(struct vas_window *window)
+static void clear_vinst_win(struct vas_window *window)
 {
int id = window->winid;
struct vas_instance *vinst = window->vinst;
@@ -879,11 +887,92 @@ struct vas_window *vas_rx_win_open(int vasid, enum 
vas_cop_type cop,
 }
 EXPORT_SYMBOL_GPL(vas_rx_win_open);
 
-/* stub for now */
+static void poll_window_busy_state(struct vas_window *window)
+{
+   int busy;
+   uint64_t val;
+
+retry:
+   /*
+* Poll Window Busy flag
+*/
+   val = read_hvwc_reg(window, VREG(WIN_STATUS));
+   busy = GET_FIELD(VAS_WIN_BUSY, val);
+   if (busy) {
+   val = 0;
+   schedule_timeout(2000);
+   goto retry;
+   }
+}
+
+static void poll_window_castout(struct vas_window *window)
+{
+   int cached;
+   uint64_t val;
+
+   /* Cast window context out of the cache */
+retry:
+   val = read_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL));
+   cached = GET_FIELD(VAS_WIN_CACHE_STATUS, val);
+   if (cached) {
+   val = 0ULL;
+   val = SET_FIELD(VAS_CASTOUT_REQ, val, 1);
+   val = SET_FIELD(VAS_PUSH_TO_MEM, val, 0);
+   write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);
+
+   schedule_timeout(2000);
+   goto retry;
+   }
+}
+
+/*
+ * Close a window.
+ *
+ * See Section 1.12.1 of VAS workbook v1.05 for details on closing window:
+ * - Disable new paste operations (unmap paste address)
+ * - Poll for the "Window Busy" bit to be cleared
+ * - Clear the Open/Enable bit for the Window.
+ * - Poll for return of window Credits (implies FIFO empty for Rx win?)
+ * - Unpin and cast window context out of cache
+ *
+ * Besides the hardware, kernel has some bookkeeping of course.
+ */
 int vas_win_close(struct vas_window *window)
 {
-   return -1;
+   uint64_t val;
+
+   if (!window)
+   return 0;
+
+   if (!window->tx_win &&