[PATCH v2 09/12] uprobes: batch uprobes_treelock during registration

[Andrii Nakryiko]

Now that we have a good separate of each registration step, take
uprobes_treelock just once for relevant registration step, and then
process all relevant uprobes in one go.

Even if writer lock introduces a relatively large delay (as might happen
with per-CPU RW semaphore), this will keep overall batch attachment
reasonably fast.

We teach put_uprobe(), though __put_uprobe() helper, to optionally take
or not uprobes_treelock, to accommodate this pattern.

With these changes we don't need insert_uprobe() operation that
unconditionally takes uprobes_treelock, so get rid of it, leaving only
lower-level __insert_uprobe() helper.

Signed-off-by: Andrii Nakryiko <and...@kernel.org>
---
 kernel/events/uprobes.c | 45 +++++++++++++++++++++--------------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 128677ffe662..ced85284bbf4 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -665,7 +665,7 @@ static void uprobe_free_rcu(struct rcu_head *rcu)
        kfree(uprobe);
 }
 
-static void put_uprobe(struct uprobe *uprobe)
+static void __put_uprobe(struct uprobe *uprobe, bool tree_locked)
 {
        s64 v;
 
@@ -683,7 +683,8 @@ static void put_uprobe(struct uprobe *uprobe)
        if (unlikely((u32)v == 0)) {
                bool destroy;
 
-               write_lock(&uprobes_treelock);
+               if (!tree_locked)
+                       write_lock(&uprobes_treelock);
                /*
                 * We might race with find_uprobe()->__get_uprobe() executed
                 * from inside read-locked uprobes_treelock, which can bump
@@ -706,7 +707,8 @@ static void put_uprobe(struct uprobe *uprobe)
                destroy = atomic64_read(&uprobe->ref) == v;
                if (destroy && uprobe_is_active(uprobe))
                        rb_erase(&uprobe->rb_node, &uprobes_tree);
-               write_unlock(&uprobes_treelock);
+               if (!tree_locked)
+                       write_unlock(&uprobes_treelock);
 
                /*
                 * Beyond here we don't need RCU protection, we are either the
@@ -745,6 +747,11 @@ static void put_uprobe(struct uprobe *uprobe)
        rcu_read_unlock_trace();
 }
 
+static void put_uprobe(struct uprobe *uprobe)
+{
+       __put_uprobe(uprobe, false);
+}
+
 static __always_inline
 int uprobe_cmp(const struct inode *l_inode, const loff_t l_offset,
               const struct uprobe *r)
@@ -844,21 +851,6 @@ static struct uprobe *__insert_uprobe(struct uprobe 
*uprobe)
        return u;
 }
 
-/*
- * Acquire uprobes_treelock and insert uprobe into uprobes_tree
- * (or reuse existing one, see __insert_uprobe() comments above).
- */
-static struct uprobe *insert_uprobe(struct uprobe *uprobe)
-{
-       struct uprobe *u;
-
-       write_lock(&uprobes_treelock);
-       u = __insert_uprobe(uprobe);
-       write_unlock(&uprobes_treelock);
-
-       return u;
-}
-
 static void
 ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe)
 {
@@ -1318,6 +1310,8 @@ int uprobe_register_batch(struct inode *inode, int cnt,
                uc->uprobe = uprobe;
        }
 
+       ret = 0;
+       write_lock(&uprobes_treelock);
        for (i = 0; i < cnt; i++) {
                struct uprobe *cur_uprobe;
 
@@ -1325,19 +1319,24 @@ int uprobe_register_batch(struct inode *inode, int cnt,
                uprobe = uc->uprobe;
 
                /* add to uprobes_tree, sorted on inode:offset */
-               cur_uprobe = insert_uprobe(uprobe);
+               cur_uprobe = __insert_uprobe(uprobe);
                /* a uprobe exists for this inode:offset combination */
                if (cur_uprobe != uprobe) {
                        if (cur_uprobe->ref_ctr_offset != 
uprobe->ref_ctr_offset) {
                                ref_ctr_mismatch_warn(cur_uprobe, uprobe);
-                               put_uprobe(cur_uprobe);
+
+                               __put_uprobe(cur_uprobe, true);
                                ret = -EINVAL;
-                               goto cleanup_uprobes;
+                               goto unlock_treelock;
                        }
                        kfree(uprobe);
                        uc->uprobe = cur_uprobe;
                }
        }
+unlock_treelock:
+       write_unlock(&uprobes_treelock);
+       if (ret)
+               goto cleanup_uprobes;
 
        for (i = 0; i < cnt; i++) {
                uc = get_uprobe_consumer(i, ctx);
@@ -1367,13 +1366,15 @@ int uprobe_register_batch(struct inode *inode, int cnt,
        }
 cleanup_uprobes:
        /* put all the successfully allocated/reused uprobes */
+       write_lock(&uprobes_treelock);
        for (i = 0; i < cnt; i++) {
                uc = get_uprobe_consumer(i, ctx);
 
                if (uc->uprobe)
-                       put_uprobe(uc->uprobe);
+                       __put_uprobe(uc->uprobe, true);
                uc->uprobe = NULL;
        }
+       write_unlock(&uprobes_treelock);
        return ret;
 }
 
-- 
2.43.0