Create per-task io poll context for both IO submission and poll task if the queue is bio based and supports polling.
This io polling context includes two queues: submission queue(sq) for storing HIPRI bio submission result(cookie) and the bio, written by submission task and read by poll task; polling queue(pq) for holding data moved from sq, only used in poll context for running bio polling. Following patches will support bio poll. Signed-off-by: Ming Lei <ming....@redhat.com> --- block/blk-core.c | 71 ++++++++++++++++++++++++++++++++------- block/blk-ioc.c | 1 + block/blk-mq.c | 14 ++++++++ block/blk.h | 46 +++++++++++++++++++++++++ include/linux/iocontext.h | 2 ++ 5 files changed, 122 insertions(+), 12 deletions(-) diff --git a/block/blk-core.c b/block/blk-core.c index d58f8a0c80de..0b00c21cbefb 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -792,16 +792,59 @@ static inline blk_status_t blk_check_zone_append(struct request_queue *q, return BLK_STS_OK; } -static inline void blk_create_io_context(struct request_queue *q) +static inline struct blk_bio_poll_ctx *blk_get_bio_poll_ctx(void) { - /* - * Various block parts want %current->io_context, so allocate it up - * front rather than dealing with lots of pain to allocate it only - * where needed. This may fail and the block layer knows how to live - * with it. - */ - if (unlikely(!current->io_context)) - create_task_io_context(current, GFP_ATOMIC, q->node); + struct io_context *ioc = current->io_context; + + return ioc ? ioc->data : NULL; +} + +static inline unsigned int bio_grp_list_size(unsigned int nr_grps) +{ + return sizeof(struct bio_grp_list) + nr_grps * + sizeof(struct bio_grp_list_data); +} + +static void bio_poll_ctx_init(struct blk_bio_poll_ctx *pc) +{ + pc->sq = (void *)pc + sizeof(*pc); + pc->sq->max_nr_grps = BLK_BIO_POLL_SQ_SZ; + + pc->pq = (void *)pc->sq + bio_grp_list_size(BLK_BIO_POLL_SQ_SZ); + pc->pq->max_nr_grps = BLK_BIO_POLL_PQ_SZ; + + spin_lock_init(&pc->sq_lock); + mutex_init(&pc->pq_lock); +} + +void bio_poll_ctx_alloc(struct io_context *ioc) +{ + struct blk_bio_poll_ctx *pc; + unsigned int size = sizeof(*pc) + + bio_grp_list_size(BLK_BIO_POLL_SQ_SZ) + + bio_grp_list_size(BLK_BIO_POLL_PQ_SZ); + + pc = kzalloc(GFP_ATOMIC, size); + if (pc) { + bio_poll_ctx_init(pc); + if (cmpxchg(&ioc->data, NULL, (void *)pc)) + kfree(pc); + } +} + +static inline bool blk_queue_support_bio_poll(struct request_queue *q) +{ + return !queue_is_mq(q) && blk_queue_poll(q); +} + +static inline void blk_bio_poll_preprocess(struct request_queue *q, + struct bio *bio) +{ + if (!(bio->bi_opf & REQ_HIPRI)) + return; + + if (!blk_queue_poll(q) || (!queue_is_mq(q) && !blk_get_bio_poll_ctx())) + bio->bi_opf &= ~REQ_HIPRI; } static noinline_for_stack bool submit_bio_checks(struct bio *bio) @@ -848,10 +891,14 @@ static noinline_for_stack bool submit_bio_checks(struct bio *bio) } } - blk_create_io_context(q); + /* + * Created per-task io poll queue if we supports bio polling + * and it is one HIPRI bio. + */ + blk_create_io_context(q, blk_queue_support_bio_poll(q) && + (bio->bi_opf & REQ_HIPRI)); - if (!blk_queue_poll(q)) - bio->bi_opf &= ~REQ_HIPRI; + blk_bio_poll_preprocess(q, bio); switch (bio_op(bio)) { case REQ_OP_DISCARD: diff --git a/block/blk-ioc.c b/block/blk-ioc.c index b0cde18c4b8c..5574c398eff6 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -19,6 +19,7 @@ static struct kmem_cache *iocontext_cachep; static inline void free_io_context(struct io_context *ioc) { + kfree(ioc->data); kmem_cache_free(iocontext_cachep, ioc); } diff --git a/block/blk-mq.c b/block/blk-mq.c index 63c81df3b8b5..c832faa52ca0 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -3852,6 +3852,17 @@ static bool blk_mq_poll_hybrid(struct request_queue *q, return blk_mq_poll_hybrid_sleep(q, rq); } +static int blk_bio_poll(struct request_queue *q, blk_qc_t cookie, bool spin) +{ + /* + * Create poll queue for storing poll bio and its cookie from + * submission queue + */ + blk_create_io_context(q, true); + + return 0; +} + /** * blk_poll - poll for IO completions * @q: the queue @@ -3875,6 +3886,9 @@ int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin) if (current->plug) blk_flush_plug_list(current->plug, false); + if (!queue_is_mq(q)) + return blk_bio_poll(q, cookie, spin); + hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; /* diff --git a/block/blk.h b/block/blk.h index 3b53e44b967e..ae58a706327e 100644 --- a/block/blk.h +++ b/block/blk.h @@ -357,4 +357,50 @@ int bio_add_hw_page(struct request_queue *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset, unsigned int max_sectors, bool *same_page); +/* grouping bios belonging to same group into one list */ +struct bio_grp_list_data { + /* group data */ + void *grp_data; + + /* all bios in this list share same 'grp_data' */ + struct bio_list list; +}; + +struct bio_grp_list { + unsigned int max_nr_grps, nr_grps; + struct bio_grp_list_data head[0]; +}; + +struct blk_bio_poll_ctx { + spinlock_t sq_lock; + struct bio_grp_list *sq; + + struct mutex pq_lock; + struct bio_grp_list *pq; +}; + +#define BLK_BIO_POLL_SQ_SZ 32U +#define BLK_BIO_POLL_PQ_SZ (BLK_BIO_POLL_SQ_SZ * 2) + +void bio_poll_ctx_alloc(struct io_context *ioc); + +static inline void blk_create_io_context(struct request_queue *q, + bool need_poll_ctx) +{ + struct io_context *ioc; + + /* + * Various block parts want %current->io_context, so allocate it up + * front rather than dealing with lots of pain to allocate it only + * where needed. This may fail and the block layer knows how to live + * with it. + */ + if (unlikely(!current->io_context)) + create_task_io_context(current, GFP_ATOMIC, q->node); + + ioc = current->io_context; + if (need_poll_ctx && unlikely(ioc && !ioc->data)) + bio_poll_ctx_alloc(ioc); +} + #endif /* BLK_INTERNAL_H */ diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index 0a9dc40b7be8..f9a467571356 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h @@ -110,6 +110,8 @@ struct io_context { struct io_cq __rcu *icq_hint; struct hlist_head icq_list; + void *data; + struct work_struct release_work; }; -- 2.29.2 -- dm-devel mailing list dm-devel@redhat.com https://listman.redhat.com/mailman/listinfo/dm-devel