Author: mjg
Date: Thu Oct  5 19:18:02 2017
New Revision: 324335
URL: https://svnweb.freebsd.org/changeset/base/324335

Log:
  locks: take the number of readers into account when waiting
  
  Previous code would always spin once before checking the lock. But a lock
  with e.g. 6 readers is not going to become free in the duration of once spin
  even if they start draining immediately.
  
  Conservatively perform one for each reader.
  
  Note that the total number of allowed spins is still extremely small and is
  subject to change later.
  
  MFC after:    1 week

Modified:
  head/sys/kern/kern_rwlock.c
  head/sys/kern/kern_sx.c
  head/sys/sys/lock.h

Modified: head/sys/kern/kern_rwlock.c
==============================================================================
--- head/sys/kern/kern_rwlock.c Thu Oct  5 19:11:25 2017        (r324334)
+++ head/sys/kern/kern_rwlock.c Thu Oct  5 19:18:02 2017        (r324335)
@@ -414,7 +414,7 @@ __rw_rlock_hard(volatile uintptr_t *c, struct thread *
 #ifdef ADAPTIVE_RWLOCKS
        volatile struct thread *owner;
        int spintries = 0;
-       int i;
+       int i, n;
 #endif
 #ifdef LOCK_PROFILING
        uint64_t waittime = 0;
@@ -488,8 +488,9 @@ __rw_rlock_hard(volatile uintptr_t *c, struct thread *
                        KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread),
                            "spinning", "lockname:\"%s\"",
                            rw->lock_object.lo_name);
-                       for (i = 0; i < rowner_loops; i++) {
-                               cpu_spinwait();
+                       for (i = 0; i < rowner_loops; i += n) {
+                               n = RW_READERS(v);
+                               lock_delay_spin(n);
                                v = RW_READ_VALUE(rw);
                                if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(td, 
v))
                                        break;
@@ -830,7 +831,7 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, ui
 #ifdef ADAPTIVE_RWLOCKS
        volatile struct thread *owner;
        int spintries = 0;
-       int i;
+       int i, n;
 #endif
        uintptr_t x;
 #ifdef LOCK_PROFILING
@@ -928,8 +929,9 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, ui
                        KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread),
                            "spinning", "lockname:\"%s\"",
                            rw->lock_object.lo_name);
-                       for (i = 0; i < rowner_loops; i++) {
-                               cpu_spinwait();
+                       for (i = 0; i < rowner_loops; i += n) {
+                               n = RW_READERS(v);
+                               lock_delay_spin(n);
                                v = RW_READ_VALUE(rw);
                                if ((v & RW_LOCK_WRITE_SPINNER) == 0)
                                        break;

Modified: head/sys/kern/kern_sx.c
==============================================================================
--- head/sys/kern/kern_sx.c     Thu Oct  5 19:11:25 2017        (r324334)
+++ head/sys/kern/kern_sx.c     Thu Oct  5 19:18:02 2017        (r324335)
@@ -502,7 +502,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t t
        GIANT_DECLARE;
 #ifdef ADAPTIVE_SX
        volatile struct thread *owner;
-       u_int i, spintries = 0;
+       u_int i, n, spintries = 0;
 #endif
 #ifdef LOCK_PROFILING
        uint64_t waittime = 0;
@@ -600,12 +600,13 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t t
                                    "lockname:\"%s\"", sx->lock_object.lo_name);
                                GIANT_SAVE();
                                spintries++;
-                               for (i = 0; i < asx_loops; i++) {
+                               for (i = 0; i < asx_loops; i += n) {
                                        if (LOCK_LOG_TEST(&sx->lock_object, 0))
                                                CTR4(KTR_LOCK,
                                    "%s: shared spinning on %p with %u and %u",
                                                    __func__, sx, spintries, i);
-                                       cpu_spinwait();
+                                       n = SX_SHARERS(x);
+                                       lock_delay_spin(n);
                                        x = SX_READ_VALUE(sx);
                                        if ((x & SX_LOCK_SHARED) == 0 ||
                                            SX_SHARERS(x) == 0)

Modified: head/sys/sys/lock.h
==============================================================================
--- head/sys/sys/lock.h Thu Oct  5 19:11:25 2017        (r324334)
+++ head/sys/sys/lock.h Thu Oct  5 19:18:02 2017        (r324335)
@@ -226,6 +226,13 @@ lock_delay_arg_init(struct lock_delay_arg *la, struct 
        la->spin_cnt = 0;
 }
 
+#define lock_delay_spin(n)     do {    \
+       u_int _i;                       \
+                                       \
+       for (_i = (n); _i > 0; _i--)    \
+               cpu_spinwait();         \
+} while (0)
+
 #define        LOCK_DELAY_SYSINIT(func) \
        SYSINIT(func##_ld, SI_SUB_LOCK, SI_ORDER_ANY, func, NULL)
 
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"

Reply via email to