On 11/10/26 12:46, Andrey Zhadchenko wrote:
queue_work() is pretty costly under high load. It takes pool->lock;
unbound pools do not track running workers (WORKER_UNBOUND is part
of WORKER_NOT_RUNNING), so an idle kworker is woken (idle exit + IPI +
context switch) even if a running worker would pick the work up right
away. Every qio has its own work, so this is paid for every I/O. With
several submitters this ends up in several context switches per I/O
and in contention on pool->lock.
Batch requests with percpu llists. Additionally track if the worker
is busy and queue_work() only if it is not. The worker processes one
batch per run and requeues itself if more qios came meanwhile, so
flush_workqueue(), e.g. on backward merge completion with I/O still
running, does not wait for a worker which is refilled all the time.
The fast path is now serial per submitting CPU: qios queued from one
CPU are processed one by one by its work, while qios from different
CPUs are still processed in parallel. This is fine, since a fast path
qio is only mapped and submitted as direct AIO: O_DIRECT is enforced
for every delta. The exception is a partial discard, which punches
the hole synchronously in the worker, so the qios queued after it on
that CPU wait. Most of them would wait anyway: on ext4 and xfs punch
hole takes the inode lock exclusively, and direct I/O submission to
the same image sleeps on it.
https://virtuozzo.atlassian.net/browse/VSTOR-95494
Feature: dm-qcow2: block device over QCOW2 files driver
Signed-off-by: Andrey Zhadchenko <[email protected]>
---
v2: moved before "declare DM_TARGET_NOWAIT"; process one batch per run
and requeue, so flush_workqueue() does not wait for a work which is
refilled all the time; commit message: fixed the claim about
re-queued qios, describes that the fast path is serial per
submitting CPU and that partial discards punch the hole in the
worker, measurements moved to the thread
drivers/md/dm-qcow2-map.c | 100 +++++++++++++++++++++++++++++++----
drivers/md/dm-qcow2-target.c | 10 +++-
drivers/md/dm-qcow2.h | 6 ++-
3 files changed, 103 insertions(+), 13 deletions(-)
diff --git a/drivers/md/dm-qcow2-map.c b/drivers/md/dm-qcow2-map.c
index 7a8deba4b0a83..a87a396af56d0 100644
--- a/drivers/md/dm-qcow2-map.c
+++ b/drivers/md/dm-qcow2-map.c
@@ -87,6 +87,7 @@ static void process_read_qio(struct qcow2 *qcow2, struct qio
*qio,
struct qcow2_map *map);
static void init_qrq_and_embedded_qio(struct qcow2_target *tgt, struct
request *rq,
struct qcow2_rq *qrq, struct qio *qio);
+static void qcow2_fast_submit(struct qcow2_target *tgt, struct qio *qio);
static loff_t bytes_off_in_cluster(struct qcow2 *qcow2, struct qio *qio)
{
@@ -362,8 +363,6 @@ struct qio *qcow2_alloc_qio(mempool_t *pool, bool zero)
}
ALLOW_ERROR_INJECTION(qcow2_alloc_qio, NULL);
-static void do_qio_work(struct work_struct *ws);
-
void qcow2_init_qio(struct qio *qio, unsigned int bi_op, struct qcow2 *qcow2)
{
qio->qcow2 = qcow2;
@@ -376,7 +375,6 @@ void qcow2_init_qio(struct qio *qio, unsigned int bi_op,
struct qcow2 *qcow2)
qio->ref_index = REF_INDEX_INVALID;
qio->state = QIO_STATE_START;
atomic_set(&qio->remaining, 1);
- INIT_WORK(&qio->worker, do_qio_work);
/*
* Initially set into BLK_STS_OK, while aio complete,
@@ -4511,7 +4509,7 @@ static void qcow2_submit_embedded_qio(struct qcow2_target
*tgt, struct qio *qio)
if (blk_rq_bytes(rq)) {
queue_list_id = QLIST_EMBEDDED;
- worker = &qio->worker;
+ worker = NULL;
} else {
WARN_ON_ONCE(qio->bi_op != REQ_OP_FLUSH);
queue_list_id = QLIST_FLUSH;
@@ -4534,8 +4532,13 @@ static void qcow2_submit_embedded_qio(struct
qcow2_target *tgt, struct qio *qio)
}
spin_unlock_irqrestore(&qcow2->deferred_lock, flags);
- if (queue)
+ if (!queue)
+ return;
+
+ if (worker)
queue_work(tgt->wq, worker);
+ else
+ qcow2_fast_submit(tgt, qio);
}
void qcow2_submit_embedded_qios(struct qcow2_target *tgt, struct list_head *list)
@@ -4634,15 +4637,11 @@ static void qio_try_fast(struct qio *qio, struct
list_head *deferred_qios)
list_add_tail(&qio->link, deferred_qios);
}
-static void do_qio_work(struct work_struct *ws)
+static void do_qio_work(struct qio *qio)
{
- struct qio *qio = container_of(ws, struct qio, worker);
- struct qio *i, *next;
struct qcow2 *qcow2 = qio->qcow2;
struct qcow2_target *tgt = qcow2->tgt;
- unsigned int pflags = current->flags;
-
- current->flags |= PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO;
+ struct qio *i, *next;
switch (qio->state) {
case QIO_STATE_START: {
@@ -4668,8 +4667,87 @@ static void do_qio_work(struct work_struct *ws)
qio_endio(qio);
break;
}
+}
+
+struct qcow2_fast_cpu {
+ struct llist_head qios;
+ struct work_struct work;
+ struct qcow2_target *tgt;
+ atomic_t busy;
+};
+
+static void qcow2_fast_submit(struct qcow2_target *tgt, struct qio *qio)
+{
+ struct qcow2_fast_cpu *fc = get_cpu_ptr(tgt->fast);
+
+ llist_add(&qio->fast_link, &fc->qios);
+ /* Pairs with smp_mb() in qcow2_fast_work() */
+ smp_mb();
If I’m not mistaken, the try_cmpxchg() in llist_add() is already fully
ordered, so the smp_mb() isn’t needed.
+ if (!atomic_read(&fc->busy) && atomic_cmpxchg(&fc->busy, 0, 1) == 0)
+ queue_work(tgt->wq, &fc->work);
+ put_cpu_ptr(tgt->fast);
+}
+
+static void qcow2_fast_work(struct work_struct *work)
+{
+ struct qcow2_fast_cpu *fc = container_of(work, struct qcow2_fast_cpu,
work);
+ unsigned int pflags = current->flags;
+ struct llist_node *node;
+ struct qio *qio, *next;
+
+ current->flags |= PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO;
+
+ node = llist_del_all(&fc->qios);
+ node = llist_reverse_order(node);
+ llist_for_each_entry_safe(qio, next, node, fast_link)
+ do_qio_work(qio);
current_restore_flags(pflags, PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO);
+
+ if (llist_empty(&fc->qios)) {
+ atomic_set(&fc->busy, 0);
+ /* just to be sure we look at the list after busy is posted */
+ smp_mb();
+ if (llist_empty(&fc->qios))
+ return;
+ if (atomic_cmpxchg(&fc->busy, 0, 1) != 0)
+ return;
+ }
+ queue_work(fc->tgt->wq, &fc->work);
+}
+
+void qcow2_fast_exit(struct qcow2_target *tgt)
+{
+ struct qcow2_fast_cpu *fc;
+ int cpu;
+
+ if (!tgt->fast)
+ return;
+ for_each_possible_cpu(cpu) {
+ fc = per_cpu_ptr(tgt->fast, cpu);
+ WARN_ON_ONCE(!llist_empty(&fc->qios));
+ }
+ free_percpu(tgt->fast);
+ tgt->fast = NULL;
+}
+
+int qcow2_fast_init(struct qcow2_target *tgt)
+{
+ struct qcow2_fast_cpu *fc;
+ int cpu;
+
+ tgt->fast = alloc_percpu(struct qcow2_fast_cpu);
+ if (!tgt->fast)
+ return -ENOMEM;
+ for_each_possible_cpu(cpu) {
+ fc = per_cpu_ptr(tgt->fast, cpu);
+ init_llist_head(&fc->qios);
+ INIT_WORK(&fc->work, qcow2_fast_work);
+ fc->tgt = tgt;
+ atomic_set(&fc->busy, 0);
+ }
+
+ return 0;
}
struct qio_data_llseek_hole {
diff --git a/drivers/md/dm-qcow2-target.c b/drivers/md/dm-qcow2-target.c
index 8337efc5c6238..3e85de5c8b050 100644
--- a/drivers/md/dm-qcow2-target.c
+++ b/drivers/md/dm-qcow2-target.c
@@ -253,6 +253,7 @@ static void qcow2_tgt_destroy(struct qcow2_target *tgt)
/* Now kill the queue */
destroy_workqueue(tgt->wq);
}
+ qcow2_fast_exit(tgt);
qcow2_merge_backward_cancel(tgt);
mempool_destroy(tgt->qio_pool);
@@ -477,6 +478,11 @@ static struct qcow2_target *alloc_qcow2_target(struct
dm_target *ti)
goto out_pool;
}
+ if (qcow2_fast_init(tgt)) {
+ ti->error = "Can't alloc fast path slots";
+ goto out_wq;
+ }
+
for (i = 0; i < 2; i++) {
release = i ? inflight_ref_exit1 : inflight_ref_exit0;
if (percpu_ref_init(&tgt->inflight_ref[i], release,
@@ -484,7 +490,7 @@ static struct qcow2_target *alloc_qcow2_target(struct
dm_target *ti)
if (i)
percpu_ref_exit(&tgt->inflight_ref[0]);
ti->error = "could not alloc percpu_ref";
- goto out_wq;
+ goto out_fast;
}
}
@@ -504,6 +510,8 @@ static struct qcow2_target *alloc_qcow2_target(struct dm_target *ti)
qcow2_set_service_operations(ti, false);
return tgt;
+out_fast:
+ qcow2_fast_exit(tgt);
out_wq:
destroy_workqueue(tgt->wq);
out_pool:
diff --git a/drivers/md/dm-qcow2.h b/drivers/md/dm-qcow2.h
index c044c630ca238..d6884eba2f9de 100644
--- a/drivers/md/dm-qcow2.h
+++ b/drivers/md/dm-qcow2.h
@@ -4,6 +4,7 @@
#include <linux/percpu-refcount.h>
#include <linux/device-mapper.h>
+#include <linux/llist.h>
#include <linux/fs.h>
#include <linux/eventfd.h>
@@ -192,6 +193,7 @@ struct qcow2_target {
*/
struct qcow2 *top;
struct workqueue_struct *wq;
+ struct qcow2_fast_cpu __percpu *fast;
struct completion inflight_ref_comp;
struct percpu_ref inflight_ref[2];
@@ -377,7 +379,7 @@ struct qio {
struct qcow2 *qcow2;
qcow2_endio_t endio_cb;
void *endio_cb_data;
- struct work_struct worker;
+ struct llist_node fast_link;
int state;
};
@@ -394,6 +396,8 @@ loff_t qcow2_llseek_hole(struct dm_target *ti, loff_t offset, int whence);
void do_qcow2_work(struct work_struct *ws);
void do_qcow2_fsync_work(struct work_struct *ws);
+int qcow2_fast_init(struct qcow2_target *tgt);
+void qcow2_fast_exit(struct qcow2_target *tgt);
int qcow2_alloc_and_insert_md_page(struct qcow2 *qcow2, u64 index,
struct md_page **md);
struct md_page *qcow2_md_page_find_or_postpone(struct qcow2 *qcow2,
_______________________________________________
Devel mailing list
[email protected]
https://lists.openvz.org/mailman/listinfo/devel
