This patch introduces new barrier pair light_mb() and heavy_mb() for
percpu rw semaphores.
This patch fixes a bug in percpu-rw-semaphores where a barrier was
missing in percpu_up_write.
This patch improves performance on the read path of
percpu-rw-semaphores: on non-x86 cpus, there was a smp_mb() in
percpu_up_read. This patch changes it to a compiler barrier and removes
the "#if defined(X86) ..." condition.
From: Lai Jiangshan <la...@cn.fujitsu.com>
Signed-off-by: Mikulas Patocka <mpato...@redhat.com>
---
include/linux/percpu-rwsem.h | 20 +++++++-------------
1 file changed, 7 insertions(+), 13 deletions(-)
Index: linux-3.6.3-fast/include/linux/percpu-rwsem.h
===================================================================
--- linux-3.6.3-fast.orig/include/linux/percpu-rwsem.h 2012-10-22
23:37:57.000000000 +0200
+++ linux-3.6.3-fast/include/linux/percpu-rwsem.h 2012-10-23
01:21:23.000000000 +0200
@@ -12,6 +12,9 @@ struct percpu_rw_semaphore {
struct mutex mtx;
};
+#define light_mb() barrier()
+#define heavy_mb() synchronize_sched()
+
static inline void percpu_down_read(struct percpu_rw_semaphore *p)
{
rcu_read_lock();
@@ -24,22 +27,12 @@ static inline void percpu_down_read(stru
}
this_cpu_inc(*p->counters);
rcu_read_unlock();
+ light_mb(); /* A, between read of p->locked and read of data, paired
with D */
}
static inline void percpu_up_read(struct percpu_rw_semaphore *p)
{
- /*
- * On X86, write operation in this_cpu_dec serves as a memory unlock
- * barrier (i.e. memory accesses may be moved before the write, but
- * no memory accesses are moved past the write).
- * On other architectures this may not be the case, so we need smp_mb()
- * there.
- */
-#if defined(CONFIG_X86) && (!defined(CONFIG_X86_PPRO_FENCE) &&
!defined(CONFIG_X86_OOSTORE))
- barrier();
-#else
- smp_mb();
-#endif
+ light_mb(); /* B, between read of the data and write to p->counter,
paired with C */
this_cpu_dec(*p->counters);
}
@@ -61,11 +54,12 @@ static inline void percpu_down_write(str
synchronize_rcu();
while (__percpu_count(p->counters))
msleep(1);
- smp_rmb(); /* paired with smp_mb() in percpu_sem_up_read() */
+ heavy_mb(); /* C, between read of p->counter and write to data, paired
with B */
}
static inline void percpu_up_write(struct percpu_rw_semaphore *p)
{
+ heavy_mb(); /* D, between write to data and write to p->locked, paired
with A */
p->locked = false;
mutex_unlock(&p->mtx);
}
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
