Module: xenomai-gch
Branch: for-forge
Commit: 81f02c94340ec92397dcf5f63de0b735ce65d1a6
URL:
http://git.xenomai.org/?p=xenomai-gch.git;a=commit;h=81f02c94340ec92397dcf5f63de0b735ce65d1a6
Author: Gilles Chanteperdrix <gilles.chanteperd...@xenomai.org>
Date: Sat Nov 12 22:43:13 2011 +0100
cobalt: implement deferred condition variables signalling
When signalling a condition variable, our previous and naive implementation
woke up waiting threads at the time of the pthread_cond_signal call. This has
an issue: if the woken up thread is higher priority than the signalling thread
there will be one context switch to the woken thread, which will try to grab
the mutex associated with the condition variable, go to sleep, and another
context switch will resume the signalling thread, which ultimately will release
the mutex to let the high priority thread run.
This new implementation, instead defers the actual waking up of the waiting
thread at the time of the mutex unlock. This avoids a system call for
pthread_cond_wait, and only needs one context switch to wake up a high priority
thread.
---
include/cobalt/pthread.h | 8 ---
kernel/cobalt/cond.c | 113 ++++++++++++++++++++++++++++++++++++++-------
kernel/cobalt/cond.h | 48 +++++++++++++++++++-
kernel/cobalt/internal.h | 20 ++++----
kernel/cobalt/mutex.c | 2 +
kernel/cobalt/mutex.h | 6 ++-
kernel/cobalt/syscall.c | 23 ++++++++-
lib/cobalt/cond.c | 54 ++++++++++++++++++++--
lib/cobalt/mutex.c | 5 ++-
9 files changed, 233 insertions(+), 46 deletions(-)
diff --git a/include/cobalt/pthread.h b/include/cobalt/pthread.h
index acfda12..e526f34 100644
--- a/include/cobalt/pthread.h
+++ b/include/cobalt/pthread.h
@@ -169,14 +169,6 @@ struct cobalt_condattr {
struct cobalt_cond;
-union __xeno_cond {
- pthread_cond_t native_cond;
- struct __shadow_cond {
- unsigned magic;
- struct cobalt_cond *cond;
- } shadow_cond;
-};
-
struct cobalt_threadstat {
unsigned long status;
unsigned long long xtime;
diff --git a/kernel/cobalt/cond.c b/kernel/cobalt/cond.c
index 1de5a57..ca6d255 100644
--- a/kernel/cobalt/cond.c
+++ b/kernel/cobalt/cond.c
@@ -47,22 +47,10 @@
*
*@{*/
+#include <nucleus/sys_ppd.h>
#include "mutex.h"
#include "cond.h"
-typedef struct cobalt_cond {
- unsigned magic;
- xnsynch_t synchbase;
- xnholder_t link; /* Link in cobalt_condq */
-
-#define link2cond(laddr) \
- ((cobalt_cond_t *)(((char *)laddr) - offsetof(cobalt_cond_t, link)))
-
- pthread_condattr_t attr;
- struct cobalt_mutex *mutex;
- cobalt_kqueues_t *owningq;
-} cobalt_cond_t;
-
static pthread_condattr_t default_cond_attr;
static void cond_destroy_internal(cobalt_cond_t * cond, cobalt_kqueues_t *q)
@@ -76,6 +64,10 @@ static void cond_destroy_internal(cobalt_cond_t * cond,
cobalt_kqueues_t *q)
xnpod_schedule(). */
xnsynch_destroy(&cond->synchbase);
xnlock_put_irqrestore(&nklock, s);
+#ifdef CONFIG_XENO_FASTSYNCH
+ xnheap_free(&xnsys_ppd_get(cond->attr.pshared)->sem_heap,
+ cond->pending_signals);
+#endif /* CONFIG_XENO_FASTSYNCH */
xnfree(cond);
}
@@ -108,6 +100,7 @@ int pthread_cond_init(pthread_cond_t * cnd, const
pthread_condattr_t * attr)
{
struct __shadow_cond *shadow = &((union __xeno_cond *)cnd)->shadow_cond;
xnflags_t synch_flags = XNSYNCH_PRIO | XNSYNCH_NOPIP;
+ struct xnsys_ppd *sys_ppd;
cobalt_cond_t *cond;
xnqueue_t *condq;
spl_t s;
@@ -120,11 +113,23 @@ int pthread_cond_init(pthread_cond_t * cnd, const
pthread_condattr_t * attr)
if (!cond)
return ENOMEM;
+#ifdef CONFIG_XENO_FASTSYNCH
+ sys_ppd = xnsys_ppd_get(attr->pshared);
+ cond->pending_signals = (unsigned long *)
+ xnheap_alloc(&sys_ppd->sem_heap,
+ sizeof(*(cond->pending_signals)));
+ if (!cond->pending_signals) {
+ err = EAGAIN;
+ goto err_free_cond;
+ }
+ *(cond->pending_signals) = 0;
+#endif /* CONFIG_XENO_FASTSYNCH */
+
xnlock_get_irqsave(&nklock, s);
if (attr->magic != COBALT_COND_ATTR_MAGIC) {
err = EINVAL;
- goto error;
+ goto err_free_pending_signals;
}
condq = &cobalt_kqueues(attr->pshared)->condq;
@@ -136,10 +141,17 @@ int pthread_cond_init(pthread_cond_t * cnd, const
pthread_condattr_t * attr)
if (holder == &shadow->cond->link) {
/* cond is already in the queue. */
err = EBUSY;
- goto error;
+ goto err_free_pending_signals;
}
}
+#ifdef CONFIG_XENO_FASTSYNCH
+ shadow->attr = *attr;
+ shadow->pending_signals_offset =
+ xnheap_mapped_offset(&sys_ppd->sem_heap,
+ cond->pending_signals);
+#endif /* CONFIG_XENO_FASTSYNCH */
+
shadow->magic = COBALT_COND_MAGIC;
shadow->cond = cond;
@@ -156,8 +168,14 @@ int pthread_cond_init(pthread_cond_t * cnd, const
pthread_condattr_t * attr)
return 0;
- error:
+ err_free_pending_signals:
xnlock_put_irqrestore(&nklock, s);
+#ifdef CONFIG_XENO_FASTSYNCH
+ xnheap_free(&xnsys_ppd_get(cond->attr.pshared)->sem_heap,
+ cond->pending_signals);
+ err_free_cond:
+ xnfree(cond);
+#endif
return err;
}
@@ -227,6 +245,7 @@ static inline int mutex_save_count(xnthread_t *cur,
unsigned *count_ptr)
{
cobalt_mutex_t *mutex;
+ xnholder_t *holder;
mutex = shadow->mutex;
if (!cobalt_obj_active(shadow, COBALT_MUTEX_MAGIC, struct
__shadow_mutex)
@@ -238,6 +257,14 @@ static inline int mutex_save_count(xnthread_t *cur,
*count_ptr = shadow->lockcnt;
+#ifdef CONFIG_XENO_FASTSYNCH
+ for (holder = getheadq(&mutex->conds);
+ holder; holder = nextq(&mutex->conds, holder)) {
+ struct cobalt_cond *cond = mutex_link2cond(holder);
+ if (*(cond->pending_signals))
+ cobalt_cond_deferred_signals(cond);
+ }
+#endif /* CONFIG_XENO_FASTSYNCH */
xnsynch_release(&mutex->synchbase);
/* Do not reschedule here, releasing the mutex and suspension must be
@@ -289,8 +316,19 @@ int cobalt_cond_timedwait_prologue(xnthread_t *cur,
goto unlock_and_return;
/* Bind mutex to cond. */
- if (cond->mutex == NULL)
+ if (cond->mutex == NULL) {
cond->mutex = mutex->mutex;
+ inith(&cond->mutex_link);
+ appendq(&mutex->mutex->conds, &cond->mutex_link);
+#ifdef CONFIG_XENO_FASTSYNCH
+ /* In case of previous calls to pthread_cond_signal
+ without any waiting thread */
+ *(cond->pending_signals) = 0;
+ xnarch_atomic_set_mask((unsigned long *)
+ mutex->mutex->synchbase.fastlock,
+ COBALT_MUTEX_COND_WAIT);
+#endif /* CONFIG_XENO_FASTSYNCH */
+ }
/* Wait for another thread to signal the condition. */
if (timed)
@@ -349,8 +387,16 @@ int cobalt_cond_timedwait_epilogue(xnthread_t *cur,
/* Unbind mutex and cond, if no other thread is waiting, if the job was
not already done. */
if (!xnsynch_nsleepers(&cond->synchbase)
- && cond->mutex == mutex->mutex)
+ && cond->mutex == mutex->mutex) {
cond->mutex = NULL;
+ removeq(&mutex->mutex->conds, &cond->mutex_link);
+#ifdef CONFIG_XENO_FASTSYNCH
+ if (!countq(&mutex->mutex->conds))
+ xnarch_atomic_clear_mask((unsigned long *)
+
mutex->mutex->synchbase.fastlock,
+ COBALT_MUTEX_COND_WAIT);
+#endif /* CONFIG_XENO_FASTSYNCH */
+ }
thread_cancellation_point(cur);
@@ -627,6 +673,37 @@ void cobalt_condq_cleanup(cobalt_kqueues_t *q)
xnlock_put_irqrestore(&nklock, s);
}
+#ifdef CONFIG_XENO_FASTSYNCH
+int cobalt_cond_deferred_signals(struct cobalt_cond *cond)
+{
+ unsigned long pending_signals;
+ int need_resched, i;
+
+ pending_signals = *(cond->pending_signals);
+
+ switch(pending_signals) {
+ case ~0UL:
+ need_resched =
+ xnsynch_flush(&cond->synchbase, 0) == XNSYNCH_RESCHED;
+ break;
+
+ case 0:
+ need_resched = 0;
+ break;
+
+ default:
+ for(i = 0, need_resched = 0; i < pending_signals; i++)
+ need_resched |=
+ xnsynch_wakeup_one_sleeper(&cond->synchbase)
+ != NULL;
+ }
+
+ *cond->pending_signals = 0;
+
+ return need_resched;
+}
+#endif /* CONFIG_XENO_FASTSYNCH */
+
void cobalt_cond_pkg_init(void)
{
initq(&cobalt_global_kqueues.condq);
diff --git a/kernel/cobalt/cond.h b/kernel/cobalt/cond.h
index 385e2f8..b2e8e94 100644
--- a/kernel/cobalt/cond.h
+++ b/kernel/cobalt/cond.h
@@ -20,8 +20,48 @@
#ifndef _POSIX_COND_H
#define _POSIX_COND_H
-#include <cobalt/posix.h>
+#include <pthread.h>
+
+struct cobalt_cond;
+
+union __xeno_cond {
+ pthread_cond_t native_cond;
+ struct __shadow_cond {
+ unsigned magic;
+#ifdef CONFIG_XENO_FASTSYNCH
+ struct cobalt_condattr attr;
+ union {
+ unsigned pending_signals_offset;
+ unsigned long *pending_signals;
+ };
+#endif /* CONFIG_XENO_FASTSYNCH */
+ struct cobalt_cond *cond;
+ } shadow_cond;
+};
+
+#if defined(__KERNEL__) || defined(__XENO_SIM__)
+
#include "mutex.h"
+#include "internal.h"
+
+typedef struct cobalt_cond {
+ unsigned magic;
+ xnsynch_t synchbase;
+ xnholder_t link; /* Link in cobalt_condq */
+
+#define link2cond(laddr) \
+ ((cobalt_cond_t *)(((char *)laddr) - offsetof(cobalt_cond_t, link)))
+
+ xnholder_t mutex_link;
+
+#define mutex_link2cond(laddr) \
+ ((cobalt_cond_t *)(((char *)laddr) - offsetof(cobalt_cond_t, mutex_link)))
+
+ unsigned long *pending_signals;
+ pthread_condattr_t attr;
+ struct cobalt_mutex *mutex;
+ cobalt_kqueues_t *owningq;
+} cobalt_cond_t;
int cobalt_cond_timedwait_prologue(xnthread_t *cur,
struct __shadow_cond *shadow,
@@ -34,10 +74,16 @@ int cobalt_cond_timedwait_epilogue(xnthread_t *cur,
struct __shadow_cond *shadow,
struct __shadow_mutex *mutex, unsigned count);
+#ifdef CONFIG_XENO_FASTSYNCH
+int cobalt_cond_deferred_signals(struct cobalt_cond *cond);
+#endif /* CONFIG_XENO_FASTSYNCH */
+
void cobalt_condq_cleanup(cobalt_kqueues_t *q);
void cobalt_cond_pkg_init(void);
void cobalt_cond_pkg_cleanup(void);
+#endif /* __KERNEL__ */
+
#endif /* !_POSIX_COND_H */
diff --git a/kernel/cobalt/internal.h b/kernel/cobalt/internal.h
index 4de6425..9c64204 100644
--- a/kernel/cobalt/internal.h
+++ b/kernel/cobalt/internal.h
@@ -39,16 +39,16 @@
#define COBALT_MUTEX_MAGIC COBALT_MAGIC(03)
#define COBALT_MUTEX_ATTR_MAGIC (COBALT_MAGIC(04) & ((1 << 24) - 1))
#define COBALT_COND_MAGIC COBALT_MAGIC(05)
-#define COBALT_COND_ATTR_MAGIC (COBALT_MAGIC(05) & ((1 << 24) - 1))
-#define COBALT_SEM_MAGIC COBALT_MAGIC(06)
-#define COBALT_KEY_MAGIC COBALT_MAGIC(07)
-#define COBALT_ONCE_MAGIC COBALT_MAGIC(08)
-#define COBALT_MQ_MAGIC COBALT_MAGIC(09)
-#define COBALT_MQD_MAGIC COBALT_MAGIC(0A)
-#define COBALT_INTR_MAGIC COBALT_MAGIC(0B)
-#define COBALT_NAMED_SEM_MAGIC COBALT_MAGIC(0C)
-#define COBALT_TIMER_MAGIC COBALT_MAGIC(0D)
-#define COBALT_SHM_MAGIC COBALT_MAGIC(0E)
+#define COBALT_COND_ATTR_MAGIC (COBALT_MAGIC(06) & ((1 << 24) - 1))
+#define COBALT_SEM_MAGIC COBALT_MAGIC(07)
+#define COBALT_KEY_MAGIC COBALT_MAGIC(08)
+#define COBALT_ONCE_MAGIC COBALT_MAGIC(09)
+#define COBALT_MQ_MAGIC COBALT_MAGIC(0A)
+#define COBALT_MQD_MAGIC COBALT_MAGIC(0B)
+#define COBALT_INTR_MAGIC COBALT_MAGIC(0C)
+#define COBALT_NAMED_SEM_MAGIC COBALT_MAGIC(0D)
+#define COBALT_TIMER_MAGIC COBALT_MAGIC(0E)
+#define COBALT_SHM_MAGIC COBALT_MAGIC(0F)
#define COBALT_MIN_PRIORITY XNSCHED_LOW_PRIO
#define COBALT_MAX_PRIORITY XNSCHED_HIGH_PRIO
diff --git a/kernel/cobalt/mutex.c b/kernel/cobalt/mutex.c
index 5ae4cbb..4d43f2a 100644
--- a/kernel/cobalt/mutex.c
+++ b/kernel/cobalt/mutex.c
@@ -49,6 +49,7 @@
#include <nucleus/sys_ppd.h>
#include "mutex.h"
+#include "cond.h"
pthread_mutexattr_t cobalt_default_mutex_attr;
@@ -117,6 +118,7 @@ int cobalt_mutex_init_internal(struct __shadow_mutex
*shadow,
inith(&mutex->link);
mutex->attr = *attr;
mutex->owningq = kq;
+ initq(&mutex->conds);
xnlock_get_irqsave(&nklock, s);
appendq(&kq->mutexq, &mutex->link);
diff --git a/kernel/cobalt/mutex.h b/kernel/cobalt/mutex.h
index a61b40c..18eab5a 100644
--- a/kernel/cobalt/mutex.h
+++ b/kernel/cobalt/mutex.h
@@ -19,8 +19,8 @@
#ifndef _POSIX_MUTEX_H
#define _POSIX_MUTEX_H
-#include <asm/xenomai/atomic.h>
#include <pthread.h>
+#include <asm/xenomai/atomic.h>
struct cobalt_mutex;
@@ -37,6 +37,8 @@ union __xeno_mutex {
xnarch_atomic_t *owner;
};
struct cobalt_mutexattr attr;
+
+#define COBALT_MUTEX_COND_WAIT XN_HANDLE_SPARE2
#endif /* CONFIG_XENO_FASTSYNCH */
} shadow_mutex;
};
@@ -55,6 +57,8 @@ typedef struct cobalt_mutex {
#define link2mutex(laddr) \
((cobalt_mutex_t *)(((char *)laddr) - offsetof(cobalt_mutex_t, link)))
+ xnqueue_t conds;
+
pthread_mutexattr_t attr;
cobalt_kqueues_t *owningq;
} cobalt_mutex_t;
diff --git a/kernel/cobalt/syscall.c b/kernel/cobalt/syscall.c
index dfb2520..0613bc8 100644
--- a/kernel/cobalt/syscall.c
+++ b/kernel/cobalt/syscall.c
@@ -1231,7 +1231,7 @@ static int __pthread_mutex_unlock(union __xeno_mutex
__user *u_mx)
return err;
}
-#else /* !CONFIG_XENO_FASTSYNCH */
+#else /* CONFIG_XENO_FASTSYNCH */
static int __pthread_mutex_check_init(union __xeno_mutex __user *u_mx,
const pthread_mutexattr_t __user *u_attr)
{
@@ -1385,7 +1385,11 @@ static int __pthread_mutex_timedlock(union __xeno_mutex
__user *u_mx,
static int __pthread_mutex_unlock(union __xeno_mutex __user *u_mx)
{
+ struct cobalt_mutex *mutex;
union __xeno_mutex mx;
+ xnholder_t *holder;
+ int need_resched;
+ spl_t s;
if (xnpod_root_p())
return -EPERM;
@@ -1395,12 +1399,25 @@ static int __pthread_mutex_unlock(union __xeno_mutex
__user *u_mx)
offsetof(struct __shadow_mutex, lock)))
return -EFAULT;
- if (xnsynch_release(&mx.shadow_mutex.mutex->synchbase))
+ mutex = mx.shadow_mutex.mutex;
+
+ xnlock_get_irqsave(&nklock, s);
+ need_resched = 0;
+ for (holder = getheadq(&mutex->conds);
+ holder; holder = nextq(&mutex->conds, holder)) {
+ struct cobalt_cond *cond = mutex_link2cond(holder);
+ if (*(cond->pending_signals))
+ need_resched |= cobalt_cond_deferred_signals(cond);
+ }
+ need_resched |= xnsynch_release(&mutex->synchbase) != NULL;
+
+ if (need_resched)
xnpod_schedule();
+ xnlock_put_irqrestore(&nklock, s);
return 0;
}
-#endif /* !CONFIG_XENO_FASTSYNCH */
+#endif /* CONFIG_XENO_FASTSYNCH */
static int __pthread_condattr_init(pthread_condattr_t __user *u_attr)
{
diff --git a/lib/cobalt/cond.c b/lib/cobalt/cond.c
index 1457c0e..f6daed5 100644
--- a/lib/cobalt/cond.c
+++ b/lib/cobalt/cond.c
@@ -20,10 +20,26 @@
#include <cobalt/syscall.h>
#include <pthread.h>
#include <kernel/cobalt/mutex.h>
+#include <kernel/cobalt/cond.h>
#include <kernel/cobalt/cb_lock.h>
extern int __cobalt_muxid;
+#ifdef CONFIG_XENO_FASTSYNCH
+#define COBALT_COND_MAGIC 0x86860505
+
+extern unsigned long xeno_sem_heap[2];
+
+static unsigned long *get_signalsp(struct __shadow_cond *shadow)
+{
+ if (likely(!shadow->attr.pshared))
+ return shadow->pending_signals;
+
+ return (unsigned long *)(xeno_sem_heap[1]
+ + shadow->pending_signals_offset);
+}
+#endif /* CONFIG_XENO_FASTSYNCH */
+
int __wrap_pthread_condattr_init(pthread_condattr_t *attr)
{
return -XENOMAI_SKINCALL1(__cobalt_muxid, __cobalt_condattr_init, attr);
@@ -64,11 +80,19 @@ int __wrap_pthread_condattr_setpshared(pthread_condattr_t
*attr, int pshared)
int __wrap_pthread_cond_init(pthread_cond_t * cond,
const pthread_condattr_t * attr)
{
- union __xeno_cond *_cond = (union __xeno_cond *)cond;
+ struct __shadow_cond *shadow =
+ &((union __xeno_cond *)cond)->shadow_cond;
int err;
err = -XENOMAI_SKINCALL2(__cobalt_muxid,
- __cobalt_cond_init, &_cond->shadow_cond, attr);
+ __cobalt_cond_init, shadow, attr);
+#ifdef CONFIG_XENO_FASTSYNCH
+ if (!err && !shadow->attr.pshared) {
+ shadow->pending_signals = (unsigned long *)
+ (xeno_sem_heap[0] + shadow->pending_signals_offset);
+ }
+#endif /* CONFIG_XENO_FASTSYNCH */
+
return err;
}
@@ -180,16 +204,38 @@ int __wrap_pthread_cond_timedwait(pthread_cond_t * cond,
int __wrap_pthread_cond_signal(pthread_cond_t * cond)
{
- union __xeno_cond *_cond = (union __xeno_cond *)cond;
+ struct __shadow_cond *shadow =
+ &((union __xeno_cond *)cond)->shadow_cond;
+#ifdef CONFIG_XENO_FASTSYNCH
+ unsigned long *pending_signals;
+
+ if (shadow->magic != COBALT_COND_MAGIC)
+ return EINVAL;
+
+ pending_signals = get_signalsp(shadow);
+ if (*pending_signals != ~0UL)
+ ++(*pending_signals);
+ return 0;
+#else /* !CONFIG_XENO_FASTSYNCH */
return -XENOMAI_SKINCALL1(__cobalt_muxid,
__cobalt_cond_signal, &_cond->shadow_cond);
+#endif /* !CONFIG_XENO_FASTSYNCH */
}
int __wrap_pthread_cond_broadcast(pthread_cond_t * cond)
{
- union __xeno_cond *_cond = (union __xeno_cond *)cond;
+ struct __shadow_cond *shadow =
+ &((union __xeno_cond *)cond)->shadow_cond;
+#ifdef CONFIG_XENO_FASTSYNCH
+ if (shadow->magic != COBALT_COND_MAGIC)
+ return EINVAL;
+
+ *get_signalsp(shadow) = ~0UL;
+ return 0;
+#else /* !CONFIG_XENO_FASTSYNCH */
return -XENOMAI_SKINCALL1(__cobalt_muxid,
__cobalt_cond_broadcast, &_cond->shadow_cond);
+#endif /* !CONFIG_XENO_FASTSYNCH */
}
diff --git a/lib/cobalt/mutex.c b/lib/cobalt/mutex.c
index d63e177..e244ee9 100644
--- a/lib/cobalt/mutex.c
+++ b/lib/cobalt/mutex.c
@@ -37,7 +37,7 @@ static xnarch_atomic_t *get_ownerp(struct __shadow_mutex
*shadow)
if (likely(!shadow->attr.pshared))
return shadow->owner;
- return (xnarch_atomic_t *) (xeno_sem_heap[1] + shadow->owner_offset);
+ return (xnarch_atomic_t *)(xeno_sem_heap[1] + shadow->owner_offset);
}
#endif /* CONFIG_XENO_FASTSYNCH */
@@ -391,6 +391,9 @@ int __wrap_pthread_mutex_unlock(pthread_mutex_t *mutex)
goto out;
}
+ if (unlikely(xnsynch_fast_check_spares(ownerp, COBALT_MUTEX_COND_WAIT)))
+ goto do_syscall;
+
if (likely(xnsynch_fast_release(ownerp, cur))) {
out:
cb_read_unlock(&shadow->lock, s);
_______________________________________________
Xenomai-git mailing list
Xenomai-git@gna.org
https://mail.gna.org/listinfo/xenomai-git
