[PATCH 14/21] block, aio: batch completion for bios/kiocbs
When completing a kiocb, there's some fixed overhead from touching the
kioctx's ring buffer the kiocb belongs to. Some newer high end block
devices can complete multiple IOs per interrupt, much like many network
interfaces have been for some time.
This plumbs through infrastructure so we can take advantage of multiple
completions at the interrupt level, and complete multiple kiocbs at the
same time.
Drivers have to be converted to take advantage of this, but it's a simple
change and the next patches will convert a few drivers.
To use it, an interrupt handler (or any code that completes bios or
requests) declares and initializes a struct batch_complete:
struct batch_complete batch;
batch_complete_init();
Then, instead of calling bio_endio(), it calls
bio_endio_batch(bio, err, ). This just adds the bio to a list in
the batch_complete.
At the end, it calls
batch_complete();
This completes all the bios all at once, building up a list of kiocbs;
then the list of kiocbs are completed all at once.
[a...@linux-foundation.org: fix warning]
[a...@linux-foundation.org: fs/aio.c needs bio.h, move bio_endio_batch()
declaration somewhere rational]
[a...@linux-foundation.org: fix warnings]
[minc...@kernel.org: fix build error due to bio_endio_batch]
[a...@linux-foundation.org: fix tracepoint in batch_complete()]
Signed-off-by: Kent Overstreet
Cc: Zach Brown
Cc: Felipe Balbi
Cc: Greg Kroah-Hartman
Cc: Mark Fasheh
Cc: Joel Becker
Cc: Rusty Russell
Cc: Jens Axboe
Cc: Asai Thambi S P
Cc: Selvan Mani
Cc: Sam Bradshaw
Cc: Jeff Moyer
Cc: Al Viro
Cc: Benjamin LaHaise
Cc: Theodore Ts'o
Signed-off-by: Minchan Kim
Signed-off-by: Andrew Morton
---
block/blk-core.c | 35 +---
block/blk-flush.c | 2 +-
block/blk.h| 3 +-
drivers/block/swim3.c | 2 +-
drivers/md/dm.c| 2 +-
fs/aio.c | 196 +
fs/bio.c | 49 +++
fs/direct-io.c | 12 +--
include/linux/aio.h| 24 -
include/linux/batch_complete.h | 22 +
include/linux/bio.h| 36 ++--
include/linux/blk_types.h | 1 +
include/linux/blkdev.h | 12 ++-
13 files changed, 270 insertions(+), 126 deletions(-)
create mode 100644 include/linux/batch_complete.h
diff --git a/block/blk-core.c b/block/blk-core.c
index 33c33bc..94aa4e7 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -153,7 +153,8 @@ void blk_rq_init(struct request_queue *q, struct request
*rq)
EXPORT_SYMBOL(blk_rq_init);
static void req_bio_endio(struct request *rq, struct bio *bio,
- unsigned int nbytes, int error)
+ unsigned int nbytes, int error,
+ struct batch_complete *batch)
{
if (error)
clear_bit(BIO_UPTODATE, >bi_flags);
@@ -167,7 +168,7 @@ static void req_bio_endio(struct request *rq, struct bio
*bio,
/* don't actually finish bio if it's part of flush sequence */
if (bio->bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ))
- bio_endio(bio, error);
+ bio_endio_batch(bio, error, batch);
}
void blk_dump_rq_flags(struct request *rq, char *msg)
@@ -2281,7 +2282,8 @@ EXPORT_SYMBOL(blk_fetch_request);
* %false - this request doesn't have any more data
* %true - this request has more data
**/
-bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
+bool blk_update_request(struct request *req, int error, unsigned int nr_bytes,
+ struct batch_complete *batch)
{
int total_bytes;
@@ -2337,7 +2339,7 @@ bool blk_update_request(struct request *req, int error,
unsigned int nr_bytes)
if (bio_bytes == bio->bi_size)
req->bio = bio->bi_next;
- req_bio_endio(req, bio, bio_bytes, error);
+ req_bio_endio(req, bio, bio_bytes, error, batch);
total_bytes += bio_bytes;
nr_bytes -= bio_bytes;
@@ -2390,14 +2392,15 @@ EXPORT_SYMBOL_GPL(blk_update_request);
static bool blk_update_bidi_request(struct request *rq, int error,
unsigned int nr_bytes,
- unsigned int bidi_bytes)
+ unsigned int bidi_bytes,
+ struct batch_complete *batch)
{
- if (blk_update_request(rq, error, nr_bytes))
+ if (blk_update_request(rq, error, nr_bytes, batch))
return true;
/* Bidi request must be completed as a whole */
if (unlikely(blk_bidi_rq(rq)) &&
- blk_update_request(rq->next_rq, error, bidi_bytes))
+ blk_update_request(rq->next_rq, error, bidi_bytes, batch))
return true;
if (blk_queue_add_random(rq->q))
@@ -2480,7 +2483,7 @@ static bool
[PATCH 14/21] block, aio: batch completion for bios/kiocbs
When completing a kiocb, there's some fixed overhead from touching the
kioctx's ring buffer the kiocb belongs to. Some newer high end block
devices can complete multiple IOs per interrupt, much like many network
interfaces have been for some time.
This plumbs through infrastructure so we can take advantage of multiple
completions at the interrupt level, and complete multiple kiocbs at the
same time.
Drivers have to be converted to take advantage of this, but it's a simple
change and the next patches will convert a few drivers.
To use it, an interrupt handler (or any code that completes bios or
requests) declares and initializes a struct batch_complete:
struct batch_complete batch;
batch_complete_init(batch);
Then, instead of calling bio_endio(), it calls
bio_endio_batch(bio, err, batch). This just adds the bio to a list in
the batch_complete.
At the end, it calls
batch_complete(batch);
This completes all the bios all at once, building up a list of kiocbs;
then the list of kiocbs are completed all at once.
[a...@linux-foundation.org: fix warning]
[a...@linux-foundation.org: fs/aio.c needs bio.h, move bio_endio_batch()
declaration somewhere rational]
[a...@linux-foundation.org: fix warnings]
[minc...@kernel.org: fix build error due to bio_endio_batch]
[a...@linux-foundation.org: fix tracepoint in batch_complete()]
Signed-off-by: Kent Overstreet koverstr...@google.com
Cc: Zach Brown z...@redhat.com
Cc: Felipe Balbi ba...@ti.com
Cc: Greg Kroah-Hartman gre...@linuxfoundation.org
Cc: Mark Fasheh mfas...@suse.com
Cc: Joel Becker jl...@evilplan.org
Cc: Rusty Russell ru...@rustcorp.com.au
Cc: Jens Axboe ax...@kernel.dk
Cc: Asai Thambi S P asamymuth...@micron.com
Cc: Selvan Mani sm...@micron.com
Cc: Sam Bradshaw sbrads...@micron.com
Cc: Jeff Moyer jmo...@redhat.com
Cc: Al Viro v...@zeniv.linux.org.uk
Cc: Benjamin LaHaise b...@kvack.org
Cc: Theodore Ts'o ty...@mit.edu
Signed-off-by: Minchan Kim minc...@kernel.org
Signed-off-by: Andrew Morton a...@linux-foundation.org
---
block/blk-core.c | 35 +---
block/blk-flush.c | 2 +-
block/blk.h| 3 +-
drivers/block/swim3.c | 2 +-
drivers/md/dm.c| 2 +-
fs/aio.c | 196 +
fs/bio.c | 49 +++
fs/direct-io.c | 12 +--
include/linux/aio.h| 24 -
include/linux/batch_complete.h | 22 +
include/linux/bio.h| 36 ++--
include/linux/blk_types.h | 1 +
include/linux/blkdev.h | 12 ++-
13 files changed, 270 insertions(+), 126 deletions(-)
create mode 100644 include/linux/batch_complete.h
diff --git a/block/blk-core.c b/block/blk-core.c
index 33c33bc..94aa4e7 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -153,7 +153,8 @@ void blk_rq_init(struct request_queue *q, struct request
*rq)
EXPORT_SYMBOL(blk_rq_init);
static void req_bio_endio(struct request *rq, struct bio *bio,
- unsigned int nbytes, int error)
+ unsigned int nbytes, int error,
+ struct batch_complete *batch)
{
if (error)
clear_bit(BIO_UPTODATE, bio-bi_flags);
@@ -167,7 +168,7 @@ static void req_bio_endio(struct request *rq, struct bio
*bio,
/* don't actually finish bio if it's part of flush sequence */
if (bio-bi_size == 0 !(rq-cmd_flags REQ_FLUSH_SEQ))
- bio_endio(bio, error);
+ bio_endio_batch(bio, error, batch);
}
void blk_dump_rq_flags(struct request *rq, char *msg)
@@ -2281,7 +2282,8 @@ EXPORT_SYMBOL(blk_fetch_request);
* %false - this request doesn't have any more data
* %true - this request has more data
**/
-bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
+bool blk_update_request(struct request *req, int error, unsigned int nr_bytes,
+ struct batch_complete *batch)
{
int total_bytes;
@@ -2337,7 +2339,7 @@ bool blk_update_request(struct request *req, int error,
unsigned int nr_bytes)
if (bio_bytes == bio-bi_size)
req-bio = bio-bi_next;
- req_bio_endio(req, bio, bio_bytes, error);
+ req_bio_endio(req, bio, bio_bytes, error, batch);
total_bytes += bio_bytes;
nr_bytes -= bio_bytes;
@@ -2390,14 +2392,15 @@ EXPORT_SYMBOL_GPL(blk_update_request);
static bool blk_update_bidi_request(struct request *rq, int error,
unsigned int nr_bytes,
- unsigned int bidi_bytes)
+ unsigned int bidi_bytes,
+ struct batch_complete *batch)
{
- if (blk_update_request(rq, error, nr_bytes))
+ if (blk_update_request(rq, error, nr_bytes, batch))
return true;
/* Bidi request
