Author: attilio Date: Wed Oct 31 18:07:18 2012 New Revision: 242402 URL: http://svn.freebsd.org/changeset/base/242402
Log: Rework the known mutexes to benefit about staying on their own cache line in order to avoid manual frobbing but using struct mtx_padalign. The sole exception being nvme and sxfge drivers, where the author redefined CACHE_LINE_SIZE manually, so they need to be analyzed and dealt with separately. Reviwed by: jimharris, alc Modified: head/sys/kern/kern_timeout.c head/sys/kern/sched_ule.c head/sys/vm/vm_page.c head/sys/vm/vm_page.h Modified: head/sys/kern/kern_timeout.c ============================================================================== --- head/sys/kern/kern_timeout.c Wed Oct 31 17:12:12 2012 (r242401) +++ head/sys/kern/kern_timeout.c Wed Oct 31 18:07:18 2012 (r242402) @@ -119,8 +119,8 @@ struct cc_mig_ent { * when the callout should be served. */ struct callout_cpu { - struct mtx cc_lock; - struct cc_mig_ent cc_migrating_entity __aligned(CACHE_LINE_SIZE); + struct mtx_padalign cc_lock; + struct cc_mig_ent cc_migrating_entity; struct callout *cc_callout; struct callout_tailq *cc_callwheel; struct callout_list cc_callfree; Modified: head/sys/kern/sched_ule.c ============================================================================== --- head/sys/kern/sched_ule.c Wed Oct 31 17:12:12 2012 (r242401) +++ head/sys/kern/sched_ule.c Wed Oct 31 18:07:18 2012 (r242402) @@ -228,8 +228,7 @@ struct tdq { * tdq_lock is padded to avoid false sharing with tdq_load and * tdq_cpu_idle. */ - struct mtx tdq_lock; /* run queue lock. */ - char pad[64 - sizeof(struct mtx)]; + struct mtx_padalign tdq_lock; /* run queue lock. */ struct cpu_group *tdq_cg; /* Pointer to cpu topology. */ volatile int tdq_load; /* Aggregate load. */ volatile int tdq_cpu_idle; /* cpu_idle() is active. */ @@ -292,7 +291,7 @@ static struct tdq tdq_cpu; #define TDQ_LOCK(t) mtx_lock_spin(TDQ_LOCKPTR((t))) #define TDQ_LOCK_FLAGS(t, f) mtx_lock_spin_flags(TDQ_LOCKPTR((t)), (f)) #define TDQ_UNLOCK(t) mtx_unlock_spin(TDQ_LOCKPTR((t))) -#define TDQ_LOCKPTR(t) (&(t)->tdq_lock) +#define TDQ_LOCKPTR(t) ((struct mtx *)(&(t)->tdq_lock)) static void sched_priority(struct thread *); static void sched_thread_priority(struct thread *, u_char); Modified: head/sys/vm/vm_page.c ============================================================================== --- head/sys/vm/vm_page.c Wed Oct 31 17:12:12 2012 (r242401) +++ head/sys/vm/vm_page.c Wed Oct 31 18:07:18 2012 (r242402) @@ -116,10 +116,10 @@ __FBSDID("$FreeBSD$"); */ struct vpgqueues vm_page_queues[PQ_COUNT]; -struct vpglocks vm_page_queue_lock; -struct vpglocks vm_page_queue_free_lock; +struct mtx_padalign vm_page_queue_mtx; +struct mtx_padalign vm_page_queue_free_mtx; -struct vpglocks pa_lock[PA_LOCK_COUNT]; +struct mtx_padalign pa_lock[PA_LOCK_COUNT]; vm_page_t vm_page_array; long vm_page_array_size; @@ -298,7 +298,7 @@ vm_page_startup(vm_offset_t vaddr) MTX_RECURSE); mtx_init(&vm_page_queue_free_mtx, "vm page free queue", NULL, MTX_DEF); for (i = 0; i < PA_LOCK_COUNT; i++) - mtx_init(&pa_lock[i].data, "vm page", NULL, MTX_DEF); + mtx_init(&pa_lock[i], "vm page", NULL, MTX_DEF); /* * Initialize the queue headers for the hold queue, the active queue, Modified: head/sys/vm/vm_page.h ============================================================================== --- head/sys/vm/vm_page.h Wed Oct 31 17:12:12 2012 (r242401) +++ head/sys/vm/vm_page.h Wed Oct 31 18:07:18 2012 (r242402) @@ -187,13 +187,8 @@ struct vpgqueues { extern struct vpgqueues vm_page_queues[PQ_COUNT]; -struct vpglocks { - struct mtx data; - char pad[CACHE_LINE_SIZE - sizeof(struct mtx)]; -} __aligned(CACHE_LINE_SIZE); - -extern struct vpglocks vm_page_queue_free_lock; -extern struct vpglocks pa_lock[]; +extern struct mtx_padalign vm_page_queue_free_mtx; +extern struct mtx_padalign pa_lock[]; #if defined(__arm__) #define PDRSHIFT PDR_SHIFT @@ -202,7 +197,7 @@ extern struct vpglocks pa_lock[]; #endif #define pa_index(pa) ((pa) >> PDRSHIFT) -#define PA_LOCKPTR(pa) &pa_lock[pa_index((pa)) % PA_LOCK_COUNT].data +#define PA_LOCKPTR(pa) ((struct mtx *)(&pa_lock[pa_index(pa) % PA_LOCK_COUNT])) #define PA_LOCKOBJPTR(pa) ((struct lock_object *)PA_LOCKPTR((pa))) #define PA_LOCK(pa) mtx_lock(PA_LOCKPTR(pa)) #define PA_TRYLOCK(pa) mtx_trylock(PA_LOCKPTR(pa)) @@ -235,8 +230,6 @@ extern struct vpglocks pa_lock[]; #define vm_page_lock_assert(m, a) mtx_assert(vm_page_lockptr((m)), (a)) #endif -#define vm_page_queue_free_mtx vm_page_queue_free_lock.data - /* * The vm_page's aflags are updated using atomic operations. To set or clear * these flags, the functions vm_page_aflag_set() and vm_page_aflag_clear() @@ -327,9 +320,8 @@ vm_page_t vm_phys_paddr_to_vm_page(vm_pa vm_page_t PHYS_TO_VM_PAGE(vm_paddr_t pa); -extern struct vpglocks vm_page_queue_lock; +extern struct mtx_padalign vm_page_queue_mtx; -#define vm_page_queue_mtx vm_page_queue_lock.data #define vm_page_lock_queues() mtx_lock(&vm_page_queue_mtx) #define vm_page_unlock_queues() mtx_unlock(&vm_page_queue_mtx) _______________________________________________ svn-src-head@freebsd.org mailing list http://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"