On Tue, Nov 13, 2018 at 08:42:28AM -0700, Jens Axboe wrote:
> If we're polling for IO on a device that doesn't use interrupts, then
> IO completion loop (and wake of task) is done by submitting task itself.
> If that is the case, then we don't need to enter the wake_up_process()
> function, we can simply mark ourselves as TASK_RUNNING.
>
> Signed-off-by: Jens Axboe <[email protected]>
> ---
> fs/block_dev.c | 6 ++----
> fs/iomap.c | 3 +--
> include/linux/blkdev.h | 19 +++++++++++++++++++
> 3 files changed, 22 insertions(+), 6 deletions(-)
One more for swap read:
---
diff --git a/mm/page_io.c b/mm/page_io.c
index d4d1c89bcddd..57572ff46016 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -140,7 +140,7 @@ static void end_swap_bio_read(struct bio *bio)
unlock_page(page);
WRITE_ONCE(bio->bi_private, NULL);
bio_put(bio);
- wake_up_process(waiter);
+ blk_wake_io_task(waiter);
put_task_struct(waiter);
}
--
> diff --git a/fs/block_dev.c b/fs/block_dev.c
> index 2f920c03996e..0ed9be8906a8 100644
> --- a/fs/block_dev.c
> +++ b/fs/block_dev.c
> @@ -181,8 +181,7 @@ static void blkdev_bio_end_io_simple(struct bio *bio)
> struct task_struct *waiter = bio->bi_private;
>
> WRITE_ONCE(bio->bi_private, NULL);
> - smp_wmb();
> - wake_up_process(waiter);
> + blk_wake_io_task(waiter);
> }
>
> static ssize_t
> @@ -309,8 +308,7 @@ static void blkdev_bio_end_io(struct bio *bio)
> struct task_struct *waiter = dio->waiter;
>
> WRITE_ONCE(dio->waiter, NULL);
> - smp_wmb();
> - wake_up_process(waiter);
> + blk_wake_io_task(waiter);
> }
> }
>
> diff --git a/fs/iomap.c b/fs/iomap.c
> index 7898927e758e..a182699e28db 100644
> --- a/fs/iomap.c
> +++ b/fs/iomap.c
> @@ -1526,8 +1526,7 @@ static void iomap_dio_bio_end_io(struct bio *bio)
> struct task_struct *waiter = dio->submit.waiter;
>
> WRITE_ONCE(dio->submit.waiter, NULL);
> - smp_wmb();
> - wake_up_process(waiter);
> + blk_wake_io_task(waiter);
> } else if (dio->flags & IOMAP_DIO_WRITE) {
> struct inode *inode = file_inode(dio->iocb->ki_filp);
>
> diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
> index ad8474ec8c58..d1ef8cbbea04 100644
> --- a/include/linux/blkdev.h
> +++ b/include/linux/blkdev.h
> @@ -1798,4 +1798,23 @@ static inline int blkdev_issue_flush(struct
> block_device *bdev, gfp_t gfp_mask,
>
> #endif /* CONFIG_BLOCK */
>
> +static inline void blk_wake_io_task(struct task_struct *waiter)
> +{
> + /*
> + * If we're polling, the task itself is doing the completions. For
> + * that case, we don't need to signal a wakeup, it's enough to just
> + * mark us as RUNNING.
> + */
> + if (waiter == current)
> + __set_current_state(TASK_RUNNING);
> + else {
> + /*
> + * Ensure the callers waiter store is ordered and seen
> + * by the ->bi_end_io() function.
> + */
> + smp_wmb();
> + wake_up_process(waiter);
> + }
> +}
> +
> #endif
> --
> 2.17.1
>
