Module Name: src
Committed By: ad
Date: Mon Jun 1 11:44:59 UTC 2020
Modified Files:
src/lib/libpthread: pthread.c pthread_cond.c pthread_int.h
pthread_mutex.c pthread_rwlock.c pthread_types.h
Log Message:
In the interests of reliability simplify waiter handling more and redo
condvars to manage the list of waiters with atomic ops.
To generate a diff of this commit:
cvs rdiff -u -r1.170 -r1.171 src/lib/libpthread/pthread.c
cvs rdiff -u -r1.69 -r1.70 src/lib/libpthread/pthread_cond.c
cvs rdiff -u -r1.104 -r1.105 src/lib/libpthread/pthread_int.h
cvs rdiff -u -r1.77 -r1.78 src/lib/libpthread/pthread_mutex.c
cvs rdiff -u -r1.40 -r1.41 src/lib/libpthread/pthread_rwlock.c
cvs rdiff -u -r1.23 -r1.24 src/lib/libpthread/pthread_types.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/lib/libpthread/pthread.c
diff -u src/lib/libpthread/pthread.c:1.170 src/lib/libpthread/pthread.c:1.171
--- src/lib/libpthread/pthread.c:1.170 Sat May 16 22:53:37 2020
+++ src/lib/libpthread/pthread.c Mon Jun 1 11:44:59 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread.c,v 1.170 2020/05/16 22:53:37 ad Exp $ */
+/* $NetBSD: pthread.c,v 1.171 2020/06/01 11:44:59 ad Exp $ */
/*-
* Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread.c,v 1.170 2020/05/16 22:53:37 ad Exp $");
+__RCSID("$NetBSD: pthread.c,v 1.171 2020/06/01 11:44:59 ad Exp $");
#define __EXPOSE_STACK 1
@@ -294,12 +294,11 @@ pthread__initthread(pthread_t t)
t->pt_self = t;
t->pt_magic = PT_MAGIC;
t->pt_willpark = 0;
- t->pt_unpark = 0;
+ t->pt_waiters[0] = 0;
t->pt_nwaiters = 0;
t->pt_sleepobj = NULL;
t->pt_signalled = 0;
t->pt_havespecific = 0;
- t->pt_early = NULL;
t->pt_lwpctl = &pthread__dummy_lwpctl;
memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops));
@@ -609,51 +608,32 @@ pthread__clear_waiters(pthread_t self)
{
int rv;
- /* Zero waiters or one waiter in error case (pthread_exit()). */
- if (self->pt_nwaiters == 0) {
- if (self->pt_unpark != 0 && self->pt_willpark == 0) {
- rv = (ssize_t)_lwp_unpark(self->pt_unpark, NULL);
- self->pt_unpark = 0;
- if (rv != 0 && errno != EALREADY && errno != EINTR &&
- errno != ESRCH) {
- pthread__errorfunc(__FILE__, __LINE__, __func__,
- "_lwp_unpark failed");
- }
- }
- return;
- }
+ pthread__smt_wake();
- /* One waiter or two waiters (the second being a deferred wakeup). */
- if (self->pt_nwaiters == 1) {
- if (self->pt_unpark != 0) {
- /* Fall through to multiple waiters case. */
- self->pt_waiters[1] = self->pt_unpark;
- self->pt_nwaiters = 2;
- self->pt_unpark = 0;
- } else if (self->pt_willpark) {
- /* Defer to _lwp_park(). */
- self->pt_unpark = self->pt_waiters[0];
- self->pt_nwaiters = 0;
- return;
- } else {
- /* Wake one now. */
- rv = (ssize_t)_lwp_unpark(self->pt_waiters[0], NULL);
- self->pt_nwaiters = 0;
- if (rv != 0 && errno != EALREADY && errno != EINTR &&
- errno != ESRCH) {
- pthread__errorfunc(__FILE__, __LINE__, __func__,
- "_lwp_unpark failed");
- }
- return;
+ switch (self->pt_nwaiters) {
+ case 0:
+ break;
+ case 1:
+ if (self->pt_willpark) {
+ break;
}
- }
-
- /* Multiple waiters. */
- rv = _lwp_unpark_all(self->pt_waiters, self->pt_nwaiters, NULL);
- self->pt_nwaiters = 0;
- if (rv != 0 && errno != EINTR) {
- pthread__errorfunc(__FILE__, __LINE__, __func__,
- "_lwp_unpark_all failed");
+ rv = _lwp_unpark(self->pt_waiters[0], NULL);
+ self->pt_waiters[0] = 0;
+ self->pt_nwaiters = 0;
+ if (rv != 0) {
+ pthread__errorfunc(__FILE__, __LINE__, __func__,
+ "_lwp_unpark failed");
+ }
+ break;
+ default:
+ rv = _lwp_unpark_all(self->pt_waiters, self->pt_nwaiters, NULL);
+ self->pt_waiters[0] = 0;
+ self->pt_nwaiters = 0;
+ if (rv != 0) {
+ pthread__errorfunc(__FILE__, __LINE__, __func__,
+ "_lwp_unpark_all failed");
+ }
+ break;
}
}
@@ -1115,7 +1095,7 @@ pthread__assertfunc(const char *file, in
function ? "\"" : "");
_sys_write(STDERR_FILENO, buf, (size_t)len);
- (void)_lwp_kill(_lwp_self(), SIGABRT);
+ (void)raise(SIGABRT);
_exit(1);
}
@@ -1163,16 +1143,12 @@ pthread__errorfunc(const char *file, int
* http://www.sun.com/software/whitepapers/solaris9/multithread.pdf
*/
-#define OOPS(msg) \
- pthread__errorfunc(__FILE__, __LINE__, __func__, msg)
-
int
pthread__park(pthread_t self, pthread_mutex_t *lock,
pthread_queue_t *queue, const struct timespec *abstime,
int cancelpt)
{
int rv, error;
- void *obj;
self->pt_willpark = 1;
pthread_mutex_unlock(lock);
@@ -1186,26 +1162,15 @@ pthread__park(pthread_t self, pthread_mu
* It is fine to test the value of pt_sleepobj without
* holding any locks, because:
*
- * o Only the blocking thread (this thread) ever sets them
+ * o Only the blocking thread (this thread) ever sets it
* to a non-NULL value.
*
- * o Other threads may set them NULL, but if they do so they
+ * o Other threads may set it NULL, but if they do so they
* must also make this thread return from _lwp_park.
*
* o _lwp_park, _lwp_unpark and _lwp_unpark_all are system
* calls and all make use of spinlocks in the kernel. So
- * these system calls act as full memory barriers, and will
- * ensure that the calling CPU's store buffers are drained.
- * In combination with the spinlock release before unpark,
- * this means that modification of pt_sleepobj/onq by another
- * thread will become globally visible before that thread
- * schedules an unpark operation on this thread.
- *
- * Note: the test in the while() statement dodges the park op if
- * we have already been awoken, unless there is another thread to
- * awaken. This saves a syscall - if we were already awakened,
- * the next call to _lwp_park() would need to return early in order
- * to eat the previous wakeup.
+ * these system calls act as full memory barriers.
*/
rv = 0;
do {
@@ -1213,9 +1178,13 @@ pthread__park(pthread_t self, pthread_mu
* If we deferred unparking a thread, arrange to
* have _lwp_park() restart it before blocking.
*/
+ pthread__assert(self->pt_nwaiters <= 1);
+ pthread__assert(self->pt_nwaiters != 0 ||
+ self->pt_waiters[0] == 0);
error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME,
- __UNCONST(abstime), self->pt_unpark, NULL, NULL);
- self->pt_unpark = 0;
+ __UNCONST(abstime), self->pt_waiters[0], NULL, NULL);
+ self->pt_waiters[0] = 0;
+ self->pt_nwaiters = 0;
if (error != 0) {
switch (rv = errno) {
case EINTR:
@@ -1225,7 +1194,8 @@ pthread__park(pthread_t self, pthread_mu
case ETIMEDOUT:
break;
default:
- OOPS("_lwp_park failed");
+ pthread__errorfunc(__FILE__, __LINE__,
+ __func__, "_lwp_park failed");
break;
}
}
@@ -1233,24 +1203,6 @@ pthread__park(pthread_t self, pthread_mu
if (cancelpt && self->pt_cancel)
rv = EINTR;
} while (self->pt_sleepobj != NULL && rv == 0);
-
- /*
- * If we have been awoken early but are still on the queue,
- * then remove ourself. Again, it's safe to do the test
- * without holding any locks.
- */
- if (__predict_false(self->pt_sleepobj != NULL)) {
- pthread_mutex_lock(lock);
- if ((obj = self->pt_sleepobj) != NULL) {
- PTQ_REMOVE(queue, self, pt_sleep);
- self->pt_sleepobj = NULL;
- if (obj != NULL && self->pt_early != NULL)
- (*self->pt_early)(obj);
- }
- pthread_mutex_unlock(lock);
- }
- self->pt_early = NULL;
-
return rv;
}
Index: src/lib/libpthread/pthread_cond.c
diff -u src/lib/libpthread/pthread_cond.c:1.69 src/lib/libpthread/pthread_cond.c:1.70
--- src/lib/libpthread/pthread_cond.c:1.69 Sat May 16 22:53:37 2020
+++ src/lib/libpthread/pthread_cond.c Mon Jun 1 11:44:59 2020
@@ -1,7 +1,7 @@
-/* $NetBSD: pthread_cond.c,v 1.69 2020/05/16 22:53:37 ad Exp $ */
+/* $NetBSD: pthread_cond.c,v 1.70 2020/06/01 11:44:59 ad Exp $ */
/*-
- * Copyright (c) 2001, 2006, 2007, 2008 The NetBSD Foundation, Inc.
+ * Copyright (c) 2001, 2006, 2007, 2008, 2020 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -29,24 +29,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-/*
- * We assume that there will be no contention on pthread_cond_t::ptc_lock
- * because functioning applications must call both the wait and wakeup
- * functions while holding the same application provided mutex. The
- * spinlock is present only to prevent libpthread causing the application
- * to crash or malfunction as a result of corrupted data structures, in
- * the event that the application is buggy.
- *
- * If there is contention on spinlock when real-time threads are in use,
- * it could cause a deadlock due to priority inversion: the thread holding
- * the spinlock may not get CPU time to make forward progress and release
- * the spinlock to a higher priority thread that is waiting for it.
- * Contention on the spinlock will only occur with buggy applications,
- * so at the time of writing it's not considered a major bug in libpthread.
- */
-
#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread_cond.c,v 1.69 2020/05/16 22:53:37 ad Exp $");
+__RCSID("$NetBSD: pthread_cond.c,v 1.70 2020/06/01 11:44:59 ad Exp $");
#include <stdlib.h>
#include <errno.h>
@@ -91,8 +75,7 @@ pthread_cond_init(pthread_cond_t *cond,
(attr == NULL) || (attr->ptca_magic == _PT_CONDATTR_MAGIC));
cond->ptc_magic = _PT_COND_MAGIC;
- pthread_lockinit(&cond->ptc_lock);
- PTQ_INIT(&cond->ptc_waiters);
+ cond->ptc_waiters = NULL;
cond->ptc_mutex = NULL;
if (attr && attr->ptca_private) {
cond->ptc_private = malloc(sizeof(clockid_t));
@@ -116,7 +99,7 @@ pthread_cond_destroy(pthread_cond_t *con
pthread__error(EINVAL, "Invalid condition variable",
cond->ptc_magic == _PT_COND_MAGIC);
pthread__error(EBUSY, "Destroying condition variable in use",
- cond->ptc_mutex == NULL);
+ cond->ptc_waiters == NULL);
cond->ptc_magic = _PT_COND_DEAD;
free(cond->ptc_private);
@@ -128,7 +111,7 @@ int
pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex,
const struct timespec *abstime)
{
- pthread_t self;
+ pthread_t self, next, waiters;
int retval;
clockid_t clkid = pthread_cond_getclock(cond);
@@ -149,52 +132,58 @@ pthread_cond_timedwait(pthread_cond_t *c
}
/* Note this thread as waiting on the CV. */
- pthread__spinlock(self, &cond->ptc_lock);
cond->ptc_mutex = mutex;
- PTQ_INSERT_HEAD(&cond->ptc_waiters, self, pt_sleep);
- self->pt_sleepobj = cond;
- pthread__spinunlock(self, &cond->ptc_lock);
+ self->pt_condwait = 1;
+ for (waiters = cond->ptc_waiters;; waiters = next) {
+ self->pt_condnext = waiters;
+#ifndef PTHREAD__ATOMIC_IS_MEMBAR
+ membar_producer();
+#endif
+ next = atomic_cas_ptr(&cond->ptc_waiters, waiters, self);
+ if (next == waiters)
+ break;
+ }
+
+ /* Drop the interlock */
+ self->pt_willpark = 1;
+ pthread_mutex_unlock(mutex);
+ self->pt_willpark = 0;
do {
- self->pt_willpark = 1;
- pthread_mutex_unlock(mutex);
- self->pt_willpark = 0;
- do {
- retval = _lwp_park(clkid, TIMER_ABSTIME,
- __UNCONST(abstime), self->pt_unpark, NULL, NULL);
- self->pt_unpark = 0;
- } while (retval == -1 && errno == ESRCH);
- pthread_mutex_lock(mutex);
-
- /*
- * If we have cancelled then exit. POSIX dictates that
- * the mutex must be held when we action the cancellation.
- *
- * If we absorbed a pthread_cond_signal() and cannot take
- * the wakeup, we must ensure that another thread does.
- *
- * If awoke early, we may still be on the sleep queue and
- * must remove ourself.
- */
+ pthread__assert(self->pt_nwaiters <= 1);
+ pthread__assert(self->pt_nwaiters != 0 ||
+ self->pt_waiters[0] == 0);
+ retval = _lwp_park(clkid, TIMER_ABSTIME, __UNCONST(abstime),
+ self->pt_waiters[0], NULL, NULL);
+ self->pt_waiters[0] = 0;
+ self->pt_nwaiters = 0;
if (__predict_false(retval != 0)) {
- switch (errno) {
- case EINTR:
- case EALREADY:
+ if (errno == EINTR || errno == EALREADY ||
+ errno == ESRCH) {
retval = 0;
- break;
- default:
+ } else {
retval = errno;
- break;
}
}
- if (__predict_false(self->pt_cancel | retval)) {
- pthread_cond_signal(cond);
- if (self->pt_cancel) {
- pthread__cancelled();
- }
- break;
+ } while (self->pt_condwait && !self->pt_cancel && !retval);
+
+ /*
+ * If we have cancelled then exit. POSIX dictates that
+ * the mutex must be held when we action the cancellation.
+ *
+ * If we absorbed a pthread_cond_signal() and cannot take
+ * the wakeup, we must ensure that another thread does.
+ *
+ * If awoke early, we may still be on the waiter list and
+ * must remove ourself.
+ */
+ pthread_mutex_lock(mutex);
+ if (__predict_false(self->pt_condwait | self->pt_cancel | retval)) {
+ pthread_cond_broadcast(cond);
+ if (self->pt_cancel) {
+ pthread__cancelled();
}
- } while (self->pt_sleepobj != NULL);
+ }
return retval;
}
@@ -208,121 +197,89 @@ pthread_cond_wait(pthread_cond_t *cond,
return pthread_cond_timedwait(cond, mutex, NULL);
}
-static int __noinline
-pthread__cond_wake_one(pthread_cond_t *cond)
+int
+pthread_cond_signal(pthread_cond_t *cond)
{
- pthread_t self, signaled;
+ pthread_t self, thread, next;
pthread_mutex_t *mutex;
- lwpid_t lid;
- /*
- * Pull the first thread off the queue. If the current thread
- * is associated with the condition variable, remove it without
- * awakening (error case in pthread_cond_timedwait()).
- */
- self = pthread__self();
- pthread__spinlock(self, &cond->ptc_lock);
- if (self->pt_sleepobj == cond) {
- PTQ_REMOVE(&cond->ptc_waiters, self, pt_sleep);
- self->pt_sleepobj = NULL;
- }
- signaled = PTQ_FIRST(&cond->ptc_waiters);
- if (__predict_false(signaled == NULL)) {
- cond->ptc_mutex = NULL;
- pthread__spinunlock(self, &cond->ptc_lock);
- return 0;
- }
+ if (__predict_false(__uselibcstub))
+ return __libc_cond_signal_stub(cond);
+
+ pthread__error(EINVAL, "Invalid condition variable",
+ cond->ptc_magic == _PT_COND_MAGIC);
+
+ /* Take ownership of one waiter. */
+ self = pthread_self();
mutex = cond->ptc_mutex;
- if (PTQ_NEXT(signaled, pt_sleep) == NULL) {
- cond->ptc_mutex = NULL;
- PTQ_INIT(&cond->ptc_waiters);
- } else {
- PTQ_REMOVE(&cond->ptc_waiters, signaled, pt_sleep);
+ for (thread = cond->ptc_waiters;; thread = next) {
+ if (thread == NULL) {
+ return 0;
+ }
+ membar_datadep_consumer(); /* for alpha */
+ next = thread->pt_condnext;
+ next = atomic_cas_ptr(&cond->ptc_waiters, thread, next);
+ if (__predict_true(next == thread)) {
+ break;
+ }
}
- signaled->pt_sleepobj = NULL;
- lid = signaled->pt_lid;
- pthread__spinunlock(self, &cond->ptc_lock);
-
- /*
- * For all valid uses of pthread_cond_signal(), the caller will
- * hold the mutex that the target is using to synchronize with.
- * To avoid the target awakening and immediately blocking on the
- * mutex, transfer the thread to be awoken to the current thread's
- * deferred wakeup list. The waiter will be set running when the
- * caller (this thread) releases the mutex.
- */
- if (__predict_false(self->pt_nwaiters == (size_t)pthread__unpark_max)) {
+ if (self->pt_nwaiters >= pthread__unpark_max) {
pthread__clear_waiters(self);
}
- self->pt_waiters[self->pt_nwaiters++] = lid;
+ self->pt_waiters[self->pt_nwaiters++] = thread->pt_lid;
+ membar_sync();
+ thread->pt_condwait = 0;
+ /* No longer safe to touch 'thread' */
pthread__mutex_deferwake(self, mutex);
return 0;
}
int
-pthread_cond_signal(pthread_cond_t *cond)
+pthread_cond_broadcast(pthread_cond_t *cond)
{
+ pthread_t self, thread, next;
+ pthread_mutex_t *mutex;
if (__predict_false(__uselibcstub))
- return __libc_cond_signal_stub(cond);
+ return __libc_cond_broadcast_stub(cond);
pthread__error(EINVAL, "Invalid condition variable",
cond->ptc_magic == _PT_COND_MAGIC);
- if (__predict_true(PTQ_EMPTY(&cond->ptc_waiters)))
+ if (cond->ptc_waiters == NULL)
return 0;
- return pthread__cond_wake_one(cond);
-}
-static int __noinline
-pthread__cond_wake_all(pthread_cond_t *cond)
-{
- pthread_t self, signaled;
- pthread_mutex_t *mutex;
- u_int max;
+ /* Take ownership of the current set of waiters. */
+ self = pthread_self();
+ mutex = cond->ptc_mutex;
+ thread = atomic_swap_ptr(&cond->ptc_waiters, NULL);
+ membar_datadep_consumer(); /* for alpha */
/*
- * Try to defer waking threads (see pthread_cond_signal()).
- * Only transfer waiters for which there is no pending wakeup.
+ * Pull waiters from the queue and add to our list. Use a memory
+ * barrier to ensure that we safely read the value of pt_condnext
+ * before 'thread' sees pt_condwait being cleared.
*/
- self = pthread__self();
- pthread__spinlock(self, &cond->ptc_lock);
- max = pthread__unpark_max;
- mutex = cond->ptc_mutex;
- PTQ_FOREACH(signaled, &cond->ptc_waiters, pt_sleep) {
- if (__predict_false(self->pt_nwaiters == max)) {
+ while (thread != NULL) {
+ if (self->pt_nwaiters >= pthread__unpark_max) {
pthread__clear_waiters(self);
}
- signaled->pt_sleepobj = NULL;
- self->pt_waiters[self->pt_nwaiters++] = signaled->pt_lid;
+ next = thread->pt_condnext;
+ self->pt_waiters[self->pt_nwaiters++] = thread->pt_lid;
+ membar_sync();
+ thread->pt_condwait = 0;
+ /* No longer safe to touch 'thread' */
+ thread = next;
}
- PTQ_INIT(&cond->ptc_waiters);
- cond->ptc_mutex = NULL;
- pthread__spinunlock(self, &cond->ptc_lock);
pthread__mutex_deferwake(self, mutex);
-
return 0;
}
int
-pthread_cond_broadcast(pthread_cond_t *cond)
-{
- if (__predict_false(__uselibcstub))
- return __libc_cond_broadcast_stub(cond);
-
- pthread__error(EINVAL, "Invalid condition variable",
- cond->ptc_magic == _PT_COND_MAGIC);
-
- if (__predict_true(PTQ_EMPTY(&cond->ptc_waiters)))
- return 0;
- return pthread__cond_wake_all(cond);
-}
-
-int
_pthread_cond_has_waiters_np(pthread_cond_t *cond)
{
- return !PTQ_EMPTY(&cond->ptc_waiters);
+ return cond->ptc_waiters != NULL;
}
int
Index: src/lib/libpthread/pthread_int.h
diff -u src/lib/libpthread/pthread_int.h:1.104 src/lib/libpthread/pthread_int.h:1.105
--- src/lib/libpthread/pthread_int.h:1.104 Sat May 16 22:53:37 2020
+++ src/lib/libpthread/pthread_int.h Mon Jun 1 11:44:59 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_int.h,v 1.104 2020/05/16 22:53:37 ad Exp $ */
+/* $NetBSD: pthread_int.h,v 1.105 2020/06/01 11:44:59 ad Exp $ */
/*-
* Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020
@@ -105,7 +105,6 @@ struct __pthread_st {
void *pt_exitval; /* Read by pthread_join() */
char *pt_name; /* Thread's name, set by the app. */
int pt_willpark; /* About to park */
- lwpid_t pt_unpark; /* Unpark this when parking */
struct pthread_lock_ops pt_lockops;/* Cached to avoid PIC overhead */
void *(*pt_func)(void *);/* Function to call at start. */
void *pt_arg; /* Argument to pass at start. */
@@ -142,10 +141,11 @@ struct __pthread_st {
volatile int pt_rwlocked; /* Handed rwlock successfully */
volatile int pt_signalled; /* Received pthread_cond_signal() */
volatile int pt_mutexwait; /* Waiting to acquire mutex */
+ volatile int pt_condwait; /* Waiting to acquire mutex */
void * volatile pt_mutexnext; /* Next thread in chain */
+ void * volatile pt_condnext; /* Next thread in chain */
void * volatile pt_sleepobj; /* Object slept on */
PTQ_ENTRY(__pthread_st) pt_sleep;
- void (*pt_early)(void *);
/* Thread-specific data. Large so it sits close to the end. */
int pt_havespecific __aligned(COHERENCY_UNIT);
Index: src/lib/libpthread/pthread_mutex.c
diff -u src/lib/libpthread/pthread_mutex.c:1.77 src/lib/libpthread/pthread_mutex.c:1.78
--- src/lib/libpthread/pthread_mutex.c:1.77 Sat May 16 22:53:37 2020
+++ src/lib/libpthread/pthread_mutex.c Mon Jun 1 11:44:59 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pthread_mutex.c,v 1.77 2020/05/16 22:53:37 ad Exp $ */
+/* $NetBSD: pthread_mutex.c,v 1.78 2020/06/01 11:44:59 ad Exp $ */
/*-
* Copyright (c) 2001, 2003, 2006, 2007, 2008, 2020 The NetBSD Foundation, Inc.
@@ -47,7 +47,7 @@
*/
#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread_mutex.c,v 1.77 2020/05/16 22:53:37 ad Exp $");
+__RCSID("$NetBSD: pthread_mutex.c,v 1.78 2020/06/01 11:44:59 ad Exp $");
#include <sys/types.h>
#include <sys/lwpctl.h>
@@ -358,26 +358,24 @@ pthread__mutex_lock_slow(pthread_mutex_t
* it is unsafe to re-enter the thread onto the waiters
* list.
*/
-#ifndef PTHREAD__ATOMIC_IS_MEMBAR
- membar_sync();
-#endif
- while (self->pt_mutexwait) {
+ do {
+ pthread__assert(self->pt_nwaiters <= 1);
+ pthread__assert(self->pt_nwaiters != 0 ||
+ self->pt_waiters[0] == 0);
error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME,
- __UNCONST(ts), self->pt_unpark, NULL, NULL);
- self->pt_unpark = 0;
- if (__predict_true(error != -1)) {
- continue;
- }
- if (errno == ETIMEDOUT && self->pt_mutexwait) {
- /*Remove self from waiters list*/
+ __UNCONST(ts), self->pt_waiters[0], NULL, NULL);
+ self->pt_waiters[0] = 0;
+ self->pt_nwaiters = 0;
+ if (error < 0 && errno == ETIMEDOUT) {
+ /* Remove self from waiters list */
pthread__mutex_wakeup(self, ptm);
- /*priority protect*/
+ /* Priority protect */
if (MUTEX_PROTECT(owner))
(void)_sched_protect(-1);
errno = serrno;
return ETIMEDOUT;
}
- }
+ } while (self->pt_mutexwait);
owner = ptm->ptm_owner;
}
}
@@ -485,7 +483,6 @@ pthread_mutex_unlock(pthread_mutex_t *pt
}
}
}
- pthread__smt_wake();
/*
* Finally, wake any waiters and return.
@@ -515,7 +512,6 @@ pthread__mutex_wakeup(pthread_t self, pt
/* Take ownership of the current set of waiters. */
thread = atomic_swap_ptr(&ptm->ptm_waiters, NULL);
membar_datadep_consumer(); /* for alpha */
- pthread__smt_wake();
/*
* Pull waiters from the queue and add to our list. Use a memory
@@ -523,23 +519,17 @@ pthread__mutex_wakeup(pthread_t self, pt
* before 'thread' sees pt_mutexwait being cleared.
*/
while (thread != NULL) {
- if (self->pt_nwaiters < pthread__unpark_max) {
- next = thread->pt_mutexnext;
- if (thread != self) {
- self->pt_waiters[self->pt_nwaiters++] =
- thread->pt_lid;
- membar_sync();
- }
- thread->pt_mutexwait = 0;
- /* No longer safe to touch 'thread' */
- thread = next;
- continue;
+ if (self->pt_nwaiters >= pthread__unpark_max) {
+ pthread__clear_waiters(self);
}
- pthread__clear_waiters(self);
- }
- if (self->pt_nwaiters > 0) {
- pthread__clear_waiters(self);
+ next = thread->pt_mutexnext;
+ self->pt_waiters[self->pt_nwaiters++] = thread->pt_lid;
+ membar_sync();
+ thread->pt_mutexwait = 0;
+ /* No longer safe to touch 'thread' */
+ thread = next;
}
+ pthread__clear_waiters(self);
}
int
Index: src/lib/libpthread/pthread_rwlock.c
diff -u src/lib/libpthread/pthread_rwlock.c:1.40 src/lib/libpthread/pthread_rwlock.c:1.41
--- src/lib/libpthread/pthread_rwlock.c:1.40 Sat May 16 22:53:37 2020
+++ src/lib/libpthread/pthread_rwlock.c Mon Jun 1 11:44:59 2020
@@ -1,7 +1,7 @@
-/* $NetBSD: pthread_rwlock.c,v 1.40 2020/05/16 22:53:37 ad Exp $ */
+/* $NetBSD: pthread_rwlock.c,v 1.41 2020/06/01 11:44:59 ad Exp $ */
/*-
- * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc.
+ * Copyright (c) 2002, 2006, 2007, 2008, 2020 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__RCSID("$NetBSD: pthread_rwlock.c,v 1.40 2020/05/16 22:53:37 ad Exp $");
+__RCSID("$NetBSD: pthread_rwlock.c,v 1.41 2020/06/01 11:44:59 ad Exp $");
#include <sys/types.h>
#include <sys/lwpctl.h>
@@ -56,7 +56,8 @@ __RCSID("$NetBSD: pthread_rwlock.c,v 1.4
static int pthread__rwlock_wrlock(pthread_rwlock_t *, const struct timespec *);
static int pthread__rwlock_rdlock(pthread_rwlock_t *, const struct timespec *);
-static void pthread__rwlock_early(void *);
+static void pthread__rwlock_early(pthread_t, pthread_rwlock_t *,
+ pthread_mutex_t *);
int _pthread_rwlock_held_np(pthread_rwlock_t *);
int _pthread_rwlock_rdheld_np(pthread_rwlock_t *);
@@ -218,10 +219,13 @@ pthread__rwlock_rdlock(pthread_rwlock_t
ptr->ptr_nreaders++;
self->pt_rwlocked = _RW_WANT_READ;
self->pt_sleepobj = &ptr->ptr_rblocked;
- self->pt_early = pthread__rwlock_early;
error = pthread__park(self, interlock, &ptr->ptr_rblocked,
ts, 0);
+ if (self->pt_sleepobj != NULL) {
+ pthread__rwlock_early(self, ptr, interlock);
+ }
+
/* Did we get the lock? */
if (self->pt_rwlocked == _RW_LOCKED) {
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
@@ -339,10 +343,13 @@ pthread__rwlock_wrlock(pthread_rwlock_t
PTQ_INSERT_TAIL(&ptr->ptr_wblocked, self, pt_sleep);
self->pt_rwlocked = _RW_WANT_WRITE;
self->pt_sleepobj = &ptr->ptr_wblocked;
- self->pt_early = pthread__rwlock_early;
error = pthread__park(self, interlock, &ptr->ptr_wblocked,
ts, 0);
+ if (self->pt_sleepobj != NULL) {
+ pthread__rwlock_early(self, ptr, interlock);
+ }
+
/* Did we get the lock? */
if (self->pt_rwlocked == _RW_LOCKED) {
#ifndef PTHREAD__ATOMIC_IS_MEMBAR
@@ -358,7 +365,6 @@ pthread__rwlock_wrlock(pthread_rwlock_t
}
}
-
int
pthread_rwlock_trywrlock(pthread_rwlock_t *ptr)
{
@@ -563,32 +569,19 @@ pthread_rwlock_unlock(pthread_rwlock_t *
* removed from the waiters lists.
*/
static void
-pthread__rwlock_early(void *obj)
+pthread__rwlock_early(pthread_t self, pthread_rwlock_t *ptr,
+ pthread_mutex_t *interlock)
{
- uintptr_t owner, set, new, next;
- pthread_rwlock_t *ptr;
- pthread_t self;
- u_int off;
+ uintptr_t owner, set, newval, next;
+ pthread_queue_t *queue;
- self = pthread__self();
-
- switch (self->pt_rwlocked) {
- case _RW_WANT_READ:
- off = offsetof(pthread_rwlock_t, ptr_rblocked);
- break;
- case _RW_WANT_WRITE:
- off = offsetof(pthread_rwlock_t, ptr_wblocked);
- break;
- default:
- pthread__errorfunc(__FILE__, __LINE__, __func__,
- "bad value of pt_rwlocked");
- off = 0;
- /* NOTREACHED */
- break;
+ pthread_mutex_lock(interlock);
+ if ((queue = self->pt_sleepobj) == NULL) {
+ pthread_mutex_unlock(interlock);
+ return;
}
-
- /* LINTED mind your own business */
- ptr = (pthread_rwlock_t *)((uint8_t *)obj - off);
+ PTQ_REMOVE(queue, self, pt_sleep);
+ self->pt_sleepobj = NULL;
owner = (uintptr_t)ptr->ptr_owner;
if ((owner & RW_THREAD) == 0) {
@@ -604,11 +597,12 @@ pthread__rwlock_early(void *obj)
set = 0;
for (;; owner = next) {
- new = (owner & ~(RW_HAS_WAITERS | RW_WRITE_WANTED)) | set;
- next = rw_cas(ptr, owner, new);
+ newval = (owner & ~(RW_HAS_WAITERS | RW_WRITE_WANTED)) | set;
+ next = rw_cas(ptr, owner, newval);
if (owner == next)
break;
}
+ pthread_mutex_unlock(interlock);
}
int
Index: src/lib/libpthread/pthread_types.h
diff -u src/lib/libpthread/pthread_types.h:1.23 src/lib/libpthread/pthread_types.h:1.24
--- src/lib/libpthread/pthread_types.h:1.23 Sat Sep 9 23:21:45 2017
+++ src/lib/libpthread/pthread_types.h Mon Jun 1 11:44:59 2020
@@ -1,7 +1,7 @@
-/* $NetBSD: pthread_types.h,v 1.23 2017/09/09 23:21:45 kamil Exp $ */
+/* $NetBSD: pthread_types.h,v 1.24 2020/06/01 11:44:59 ad Exp $ */
/*-
- * Copyright (c) 2001, 2008 The NetBSD Foundation, Inc.
+ * Copyright (c) 2001, 2008, 2020 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -172,7 +172,8 @@ struct __pthread_cond_st {
/* Protects the queue of waiters */
__pthread_spin_t ptc_lock;
- pthread_queue_t ptc_waiters;
+ pthread_t volatile ptc_waiters;
+ void *ptc_spare;
pthread_mutex_t *ptc_mutex; /* Current mutex */
void *ptc_private;
@@ -183,7 +184,8 @@ struct __pthread_cond_st {
#define _PTHREAD_COND_INITIALIZER { _PT_COND_MAGIC, \
__SIMPLELOCK_UNLOCKED, \
- {NULL, NULL}, \
+ NULL, \
+ NULL, \
NULL, \
NULL \
}
