Add percpu_rw_semaphore->recursive boolean. If it is true then the recursive percpu_down_read() is safe, percpu_down_write() doesn't exclude the new readers, like cpu_hotplug_begin().
Signed-off-by: Oleg Nesterov <o...@redhat.com> --- include/linux/percpu-rwsem.h | 15 ++++++++++----- kernel/events/uprobes.c | 2 +- kernel/locking/percpu-rwsem.c | 15 +++++++++++---- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h index 9202e73..9441abd 100644 --- a/include/linux/percpu-rwsem.h +++ b/include/linux/percpu-rwsem.h @@ -13,16 +13,18 @@ struct percpu_rw_semaphore { int state; struct rcu_sync_struct rss; wait_queue_head_t writer; + bool recursive; struct rw_semaphore rw_sem; }; -#define DEFINE_STATIC_PERCPU_RWSEM(name) \ +#define DEFINE_STATIC_PERCPU_RWSEM(name, rec) \ static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_refcount_##name); \ static struct percpu_rw_semaphore name = { \ .refcount = &__percpu_rwsem_refcount_##name, \ .state = 0, \ .rss = __RCU_SYNC_INITIALIZER(name.rss, RCU_SCHED_SYNC, 1), \ .writer = __WAIT_QUEUE_HEAD_INITIALIZER(name.writer), \ + .recursive = rec, \ .rw_sem = __RWSEM_INITIALIZER(name.rw_sem), \ } @@ -37,7 +39,10 @@ static inline void percpu_down_read(struct percpu_rw_semaphore *sem) { might_sleep(); - rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_); + if (sem->recursive) + rwlock_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_); + else + rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_); preempt_disable(); /* @@ -97,14 +102,14 @@ static inline void percpu_up_read(struct percpu_rw_semaphore *sem) extern void percpu_down_write(struct percpu_rw_semaphore *); extern void percpu_up_write(struct percpu_rw_semaphore *); -extern int __percpu_init_rwsem(struct percpu_rw_semaphore *, +extern int __percpu_init_rwsem(struct percpu_rw_semaphore *, bool, const char *, struct lock_class_key *); extern void percpu_free_rwsem(struct percpu_rw_semaphore *); -#define percpu_init_rwsem(sem) \ +#define percpu_init_rwsem(sem, recursive) \ ({ \ static struct lock_class_key rwsem_key; \ - __percpu_init_rwsem(sem, #sem, &rwsem_key); \ + __percpu_init_rwsem(sem, recursive, #sem, &rwsem_key); \ }) #endif diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index cb346f2..a4813a1 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1985,7 +1985,7 @@ static int __init init_uprobes(void) for (i = 0; i < UPROBES_HASH_SZ; i++) mutex_init(&uprobes_mmap_mutex[i]); - if (percpu_init_rwsem(&dup_mmap_sem)) + if (percpu_init_rwsem(&dup_mmap_sem, false)) return -ENOMEM; return register_die_notifier(&uprobe_exception_nb); diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index 609c13b..3db7c45 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -10,7 +10,7 @@ enum { readers_slow, readers_block }; -int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, +int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, bool recursive, const char *name, struct lock_class_key *rwsem_key) { sem->refcount = alloc_percpu(unsigned int); @@ -20,6 +20,7 @@ int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, sem->state = readers_slow; rcu_sync_init(&sem->rss, RCU_SCHED_SYNC, true); init_waitqueue_head(&sem->writer); + sem->recursive = recursive; __init_rwsem(&sem->rw_sem, name, rwsem_key); return 0; @@ -124,9 +125,15 @@ void __percpu_up_read(struct percpu_rw_semaphore *sem) */ static bool readers_active_check(struct percpu_rw_semaphore *sem) { - if (per_cpu_sum(*sem->refcount) != 0) + if (sem->recursive && !down_write_trylock(&sem->rw_sem)) return false; + if (per_cpu_sum(*sem->refcount) != 0) { + if (sem->recursive) + up_write(&sem->rw_sem); + return false; + } + /* * If we observed the decrement; ensure we see the entire critical * section. @@ -155,8 +162,8 @@ void percpu_down_write(struct percpu_rw_semaphore *sem) * then we are guaranteed to see their sem->refcount increment, and * therefore will wait for them. */ - - down_write(&sem->rw_sem); + if (!sem->recursive) + down_write(&sem->rw_sem); /* Wait for all now active readers to complete. */ wait_event(sem->writer, readers_active_check(sem)); } -- 1.5.5.1 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/