This is an automated email from the ASF dual-hosted git repository.

xiaoxiang781216 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nuttx.git

commit fccaaa5cfc6f46b173674038c695450dc2a45a21
Author: ouyangxiangzhen <[email protected]>
AuthorDate: Mon Nov 9 21:09:06 2026 +0800

    spinlock: add lock wait policy
    
    The WFE/SEV lock waiting made every unlock pay for a DSB before SEV, a
    full barrier on the unlock hot path.  Measured with the ostest spinlock
    loop on qemu-armv8a (4 CPUs), an unconditional DSB/SEV cost 155.9 ->
    11.3 M ops/s single-threaded and 26.9 -> 3.5 M ops/s with 4 threads
    versus busy wait, and the trade-off differs per system.
    
    Add a lock wait policy choice and route the spinlock paths through
    SPIN_WAIT() in the spin loop and SPIN_KICK() after the release store
    (9 call sites).
    
    SPINWAIT_RELAX (default) waits with the UP_RELAX() relax hint and
    never kicks: the unlock stays a plain release store.  Where an
    architecture defines no hint, the tier degenerates to a plain busy
    wait, which makes a separate NOP tier redundant.
    
    SPINWAIT_PM (default when CONFIG_PM is enabled) waits with WFE() and
    wakes with SEV(): the DSB before SEV is carried by the Arm UP_SEV()
    (see the previous commit, ARM DDI 0406C), so no extra barrier is
    emitted here -- an explicit UP_DSB() would double the barrier on Arm
    and pay an mfence per unlock on x86_64 (UP_DSB=mfence, UP_SEV empty).
    
    Evidence: the qemu-armv8a numbers above; under SPINWAIT_RELAX the
    arm64 unlock path contains no SEV at all, while armv7-a with
    SPINWAIT_PM keeps the WFE/SEV waiting.
    
    Assisted-By: mimo-v2.6-pro
    Signed-off-by: ouyangxiangzhen <[email protected]>
---
 include/nuttx/spinlock.h | 31 ++++++++++++++++++++++---------
 sched/Kconfig            | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/include/nuttx/spinlock.h b/include/nuttx/spinlock.h
index c4bac30e2cf..0ab876633a6 100644
--- a/include/nuttx/spinlock.h
+++ b/include/nuttx/spinlock.h
@@ -54,6 +54,19 @@ extern "C"
 #  define SPINLOCK_INLINE static inline_function
 #endif
 
+/* Lock wait policy, see "Spinlock wait policy" in sched/Kconfig.
+ * SPIN_WAIT() runs in the spin loop, SPIN_KICK() after the lock
+ * release store to wake the waiters.
+ */
+
+#ifdef CONFIG_SPINWAIT_PM
+#  define SPIN_WAIT()  UP_WFE()
+#  define SPIN_KICK()  UP_SEV()
+#else /* SPINWAIT_RELAX */
+#  define SPIN_WAIT()  UP_RELAX()
+#  define SPIN_KICK()
+#endif
+
 /****************************************************************************
  * Public Function Prototypes
  ****************************************************************************/
@@ -149,12 +162,12 @@ void spin_lock_notrace(FAR volatile spinlock_t *lock)
   int32_t ticket = atomic_add_relaxed(&lock->next, 1);
   while (atomic_read_acquire(&lock->owner) != ticket)
     {
-      UP_WFE();
+      SPIN_WAIT();
     }
 #else /* CONFIG_TICKET_SPINLOCK */
   while (atomic_xchg_acquire((FAR atomic_t *)lock, SP_LOCKED) == SP_LOCKED)
     {
-      UP_WFE();
+      SPIN_WAIT();
     }
 #endif
 }
@@ -338,7 +351,7 @@ spin_unlock_notrace(FAR volatile spinlock_t *lock)
 #else
   atomic_set_release((FAR atomic_t *)lock, SP_UNLOCKED);
 #endif
-  UP_SEV();
+  SPIN_KICK();
 }
 #else
 #  define spin_unlock_notrace(lock)
@@ -590,7 +603,7 @@ void rspin_lock(FAR rspinlock_t *lock)
               break;
             }
 
-          UP_WFE();
+          SPIN_WAIT();
         }
 
       /* The local value shares the same memory location with the
@@ -923,7 +936,7 @@ bool rspin_unlock(FAR rspinlock_t *lock)
   if (RSPINLOCK_COUNT(val) == 1u)
     {
       atomic_set_release(&lock->val, 0u);
-      UP_SEV();
+      SPIN_KICK();
       ret = true;
     }
   else
@@ -1054,7 +1067,7 @@ SPINLOCK_INLINE void read_lock(FAR volatile rwlock_t 
*lock)
 
           if (old == RW_SP_WRITE_LOCKED)
             {
-              UP_WFE();
+              SPIN_WAIT();
             }
         }
     }
@@ -1137,7 +1150,7 @@ static inline_function void read_unlock(FAR volatile 
rwlock_t *lock)
   DEBUGASSERT(atomic_read(lock) >= RW_SP_READ_LOCKED);
 
   atomic_sub_release(lock, 1);
-  UP_SEV();
+  SPIN_KICK();
 }
 
 /****************************************************************************
@@ -1180,7 +1193,7 @@ SPINLOCK_INLINE void write_lock(FAR volatile rwlock_t 
*lock)
           break;
         }
 
-      UP_WFE();
+      SPIN_WAIT();
     }
 
   nxsched_critmon_busywait(false, return_address(0));
@@ -1246,7 +1259,7 @@ static inline_function void write_unlock(FAR volatile 
rwlock_t *lock)
   DEBUGASSERT(atomic_read(lock) == RW_SP_WRITE_LOCKED);
 
   atomic_set_release(lock, RW_SP_UNLOCKED);
-  UP_SEV();
+  SPIN_KICK();
 }
 
 /****************************************************************************
diff --git a/sched/Kconfig b/sched/Kconfig
index 57c00029c12..61e88756f24 100644
--- a/sched/Kconfig
+++ b/sched/Kconfig
@@ -306,6 +306,39 @@ config SPINLOCK_INLINE
                this option to emit callable non-inline versions and reduce
                header-inline code growth.
 
+choice
+       prompt "Spinlock wait policy"
+       default SPINWAIT_PM if PM
+       default SPINWAIT_RELAX
+
+config SPINWAIT_RELAX
+       bool "Relax hint wait"
+       ---help---
+               Spinlock waiters poll the lock with the architecture relax
+               hint (PAUSE on x86_64, YIELD on aarch64) and unlocking sends
+               no wake event.  The hint reduces bus and pipeline
+               contention without sleeping, and the unlock stays a plain
+               release store.
+
+config SPINWAIT_PM
+       bool "WFE/SEV wait (power saving)"
+       ---help---
+               Spinlock waiters may sleep in the spin loop (WFE on Arm)
+               and unlocking sends a wake event (DSB+SEV).  Waiting CPUs
+               can enter a low-power state instead of hammering the
+               interconnect while a lock is held, at the cost of a more
+               expensive unlock.
+
+               On Arm, SEV is not ordered with memory accesses, so the
+               unlock needs a DSB before SEV to make the release store
+               visible before the event (ARM DDI 0406C, "Use of WFE and
+               SEV with Locks"); expect lower lock/unlock throughput.
+
+               On architectures without WFE/SEV semantics this policy
+               degrades to busy wait.
+
+endchoice
+
 endif # SPINLOCK
 
 config IRQCHAIN

Reply via email to