On Fri, May 03, 2019 at 04:50:59PM +0200, Peter Zijlstra wrote:
> So how about something like so then?
> --- a/kernel/locking/percpu-rwsem.c
> +++ b/kernel/locking/percpu-rwsem.c
> @@ -63,7 +66,7 @@ int __percpu_down_read(struct percpu_rw_
> * If !readers_block the critical section starts here, matched by the
> * release in percpu_up_write().
> */
> - if (likely(!smp_load_acquire(&sem->readers_block)))
> + if (likely(!atomic_read_acquire(&sem->block)))
> return 1;
>
> /*
> @@ -80,14 +83,8 @@ int __percpu_down_read(struct percpu_rw_
> * and reschedule on the preempt_enable() in percpu_down_read().
> */
> preempt_enable_no_resched();
> -
> - /*
> - * Avoid lockdep for the down/up_read() we already have them.
> - */
> - __down_read(&sem->rw_sem);
> + wait_event(sem->waiters, !atomic_read(&sem->block));
That should be:
wait_event(sem->waiters, !atomic_read_acquire(&sem->block));
I suppose.
> this_cpu_inc(*sem->read_count);
> - __up_read(&sem->rw_sem);
> -
> preempt_disable();
> return 1;
> }
> @@ -104,7 +101,7 @@ void __percpu_up_read(struct percpu_rw_s
> __this_cpu_dec(*sem->read_count);
>
> /* Prod writer to recheck readers_active */
> - rcuwait_wake_up(&sem->writer);
> + wake_up(&sem->waiters);
> }
> EXPORT_SYMBOL_GPL(__percpu_up_read);
>
> @@ -139,18 +136,22 @@ static bool readers_active_check(struct
> return true;
> }
>
> +static inline bool acquire_block(struct percpu_rw_semaphore *sem)
> +{
> + if (atomic_read(&sem->block))
> + return false;
> +
> + return atomic_xchg(&sem->block, 1) == 0;
> +}
> +
> void percpu_down_write(struct percpu_rw_semaphore *sem)
> {
> + rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
> +
> /* Notify readers to take the slow path. */
> rcu_sync_enter(&sem->rss);
>
> - down_write(&sem->rw_sem);
> -
> - /*
> - * Notify new readers to block; up until now, and thus throughout the
> - * longish rcu_sync_enter() above, new readers could still come in.
> - */
> - WRITE_ONCE(sem->readers_block, 1);
> + wait_event_exclusive(sem->waiters, acquire_block(sem));
>
> smp_mb(); /* D matches A */
And we can remove that smp_mb() and rely on the atomic_xchg() from
acquire_block().
>
> @@ -161,7 +162,7 @@ void percpu_down_write(struct percpu_rw_
> */
>
> /* Wait for all now active readers to complete. */
> - rcuwait_wait_event(&sem->writer, readers_active_check(sem));
> + wait_event(sem->waiters, readers_active_check(sem));
> }
> EXPORT_SYMBOL_GPL(percpu_down_write);
>
> @@ -177,12 +178,8 @@ void percpu_up_write(struct percpu_rw_se
> * Therefore we force it through the slow path which guarantees an
> * acquire and thereby guarantees the critical section's consistency.
> */
> - smp_store_release(&sem->readers_block, 0);
> -
> - /*
> - * Release the write lock, this will allow readers back in the game.
> - */
> - up_write(&sem->rw_sem);
> + atomic_set_release(&sem->block, 0);
> + wake_up(&sem->waiters);
>
> /*
> * Once this completes (at least one RCU-sched grace period hence) the
> @@ -190,5 +187,21 @@ void percpu_up_write(struct percpu_rw_se
> * exclusive write lock because its counting.
> */
> rcu_sync_exit(&sem->rss);
> +
> + rwsem_release(&sem->dep_map, 1, _RET_IP_);
> }
> EXPORT_SYMBOL_GPL(percpu_up_write);
