[This patch is a remake of https://mail.gna.org/public/adeos-main/2006-09/msg00013.html]
For the !SMP case, we can inline a few stalling services for the root domain. This required reordering ipipe_domain slightly so that cpudata[0].status is also the first field. This assumes that there was not specific optimisation involved in the previous layout, correct me if I'm wrong. Earlier measurements showed a slight performance increase of Linux over I-pipe, specifically on low-end (no functions calls, just simple bit fiddling for the stall that any arch should be able to handle with one or two ops). Jan
---
include/linux/ipipe.h | 12 ++----------
include/linux/ipipe_base.h | 39 +++++++++++++++++++++++++++++++++++++--
kernel/ipipe/core.c | 40 ++++++++++++++++++++++------------------
3 files changed, 61 insertions(+), 30 deletions(-)
Index: linux-2.6.20/include/linux/ipipe.h
===================================================================
--- linux-2.6.20.orig/include/linux/ipipe.h
+++ linux-2.6.20/include/linux/ipipe.h
@@ -53,13 +53,6 @@
#define IPIPE_SPRINTK_FLAG 0 /* Synchronous printk() allowed */
#define IPIPE_AHEAD_FLAG 1 /* Domain always heads the pipeline */
-/* Per-cpu pipeline status */
-#define IPIPE_STALL_FLAG 0 /* Stalls a pipeline stage --
guaranteed at bit #0 */
-#define IPIPE_SYNC_FLAG 1 /* The interrupt syncer is
running for the domain */
-#define IPIPE_NOSTACK_FLAG 2 /* Domain currently runs on a foreign
stack */
-
-#define IPIPE_SYNC_MASK (1 << IPIPE_SYNC_FLAG)
-
/* Interrupt control bits */
#define IPIPE_HANDLE_FLAG 0
#define IPIPE_PASS_FLAG 1
@@ -142,10 +135,8 @@ typedef int (*ipipe_event_handler_t)(uns
void *data);
struct ipipe_domain {
- struct list_head p_link; /* Link in pipeline */
-
struct ipcpudata {
- unsigned long status;
+ unsigned long status; /* Must be first in ipipe_domain */
unsigned long irq_pending_hi;
unsigned long irq_pending_lo[IPIPE_IRQ_IWORDS];
struct ipirqcnt {
@@ -162,6 +153,7 @@ struct ipipe_domain {
void *cookie;
} ____cacheline_aligned irqs[IPIPE_NR_IRQS];
+ struct list_head p_link; /* Link in pipeline */
ipipe_event_handler_t evhand[IPIPE_NR_EVENTS]; /* Event handlers. */
unsigned long long evself; /* Self-monitored event bits. */
unsigned long flags;
Index: linux-2.6.20/include/linux/ipipe_base.h
===================================================================
--- linux-2.6.20.orig/include/linux/ipipe_base.h
+++ linux-2.6.20/include/linux/ipipe_base.h
@@ -25,19 +25,54 @@
#ifdef CONFIG_IPIPE
+/* Per-cpu pipeline status */
+#define IPIPE_STALL_FLAG 0 /* Stalls a pipeline stage --
guaranteed at bit #0 */
+#define IPIPE_SYNC_FLAG 1 /* The interrupt syncer is
running for the domain */
+#define IPIPE_NOSTACK_FLAG 2 /* Domain currently runs on a foreign
stack */
+
+#define IPIPE_SYNC_MASK (1 << IPIPE_SYNC_FLAG)
+
+
extern struct ipipe_domain ipipe_root;
#define ipipe_root_domain (&ipipe_root)
-void __ipipe_stall_root(void);
+#ifdef CONFIG_SMP
-void __ipipe_unstall_root(void);
+void __ipipe_stall_root(void);
unsigned long __ipipe_test_root(void);
unsigned long __ipipe_test_and_stall_root(void);
+#else /* !CONFIG_SMP */
+
+/*
+ * Note: This cast relies on cpudata[0].status being the first element in the
+ * root domain structure (for UP only).
+ */
+#define __ipipe_root_status (unsigned long *)&ipipe_root
+
+static inline void __ipipe_stall_root(void)
+{
+ set_bit(IPIPE_STALL_FLAG, __ipipe_root_status);
+}
+
+static inline unsigned long __ipipe_test_root(void)
+{
+ return test_bit(IPIPE_STALL_FLAG, __ipipe_root_status);
+}
+
+static inline unsigned long __ipipe_test_and_stall_root(void)
+{
+ return test_and_set_bit(IPIPE_STALL_FLAG, __ipipe_root_status);
+}
+
+#endif /* !CONFIG_SMP */
+
+void __ipipe_unstall_root(void);
+
void __ipipe_restore_root(unsigned long x);
#endif /* CONFIG_IPIPE */
Index: linux-2.6.20/kernel/ipipe/core.c
===================================================================
--- linux-2.6.20.orig/kernel/ipipe/core.c
+++ linux-2.6.20/kernel/ipipe/core.c
@@ -147,6 +147,7 @@ void __ipipe_cleanup_domain(struct ipipe
#endif /* CONFIG_SMP */
}
+#ifdef CONFIG_SMP
void __ipipe_stall_root(void)
{
ipipe_declare_cpuid;
@@ -157,22 +158,6 @@ void __ipipe_stall_root(void)
ipipe_put_cpu(flags);
}
-void __ipipe_unstall_root(void)
-{
- ipipe_declare_cpuid;
-
- local_irq_disable_hw();
-
- ipipe_load_cpuid();
-
- __clear_bit(IPIPE_STALL_FLAG,
&ipipe_root_domain->cpudata[cpuid].status);
-
- if (unlikely(ipipe_root_domain->cpudata[cpuid].irq_pending_hi != 0))
- __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
-
- local_irq_enable_hw();
-}
-
unsigned long __ipipe_test_root(void)
{
unsigned long flags, x;
@@ -197,6 +182,23 @@ unsigned long __ipipe_test_and_stall_roo
return x;
}
+#endif /* CONFIG_SMP */
+
+void __ipipe_unstall_root(void)
+{
+ ipipe_declare_cpuid;
+
+ local_irq_disable_hw();
+
+ ipipe_load_cpuid();
+
+ __clear_bit(IPIPE_STALL_FLAG,
&ipipe_root_domain->cpudata[cpuid].status);
+
+ if (unlikely(ipipe_root_domain->cpudata[cpuid].irq_pending_hi != 0))
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+
+ local_irq_enable_hw();
+}
void __ipipe_restore_root(unsigned long x)
{
@@ -1395,10 +1397,12 @@ EXPORT_SYMBOL(ipipe_test_and_unstall_pip
EXPORT_SYMBOL(ipipe_unstall_pipeline_head);
EXPORT_SYMBOL(__ipipe_restore_pipeline_head);
EXPORT_SYMBOL(__ipipe_unstall_root);
-EXPORT_SYMBOL(__ipipe_stall_root);
EXPORT_SYMBOL(__ipipe_restore_root);
-EXPORT_SYMBOL(__ipipe_test_and_stall_root);
+#ifdef CONFIG_SMP
+EXPORT_SYMBOL(__ipipe_stall_root);
EXPORT_SYMBOL(__ipipe_test_root);
+EXPORT_SYMBOL(__ipipe_test_and_stall_root);
+#endif /* CONFIG_SMP */
EXPORT_SYMBOL(__ipipe_pipeline);
EXPORT_SYMBOL(ipipe_register_domain);
EXPORT_SYMBOL(ipipe_unregister_domain);
---
include/asm-i386/irqflags.h | 20 ++++++++++++++++++--
1 file changed, 18 insertions(+), 2 deletions(-)
Index: linux-2.6.20/include/asm-i386/irqflags.h
===================================================================
--- linux-2.6.20.orig/include/asm-i386/irqflags.h
+++ linux-2.6.20/include/asm-i386/irqflags.h
@@ -24,6 +24,7 @@ static inline unsigned long __raw_local_
#ifdef CONFIG_IPIPE
flags = (!__ipipe_test_root()) << 9;
+ barrier();
#else
__asm__ __volatile__(
"pushfl ; popl %0"
@@ -38,6 +39,7 @@ static inline unsigned long __raw_local_
static inline void raw_local_irq_restore(unsigned long flags)
{
#ifdef CONFIG_IPIPE
+ barrier();
__ipipe_restore_root(!(flags & 0x200));
#else
__asm__ __volatile__(
@@ -53,6 +55,7 @@ static inline void raw_local_irq_disable
{
#ifdef CONFIG_IPIPE
__ipipe_stall_root();
+ barrier();
#else
__asm__ __volatile__("cli" : : : "memory");
#endif
@@ -61,6 +64,7 @@ static inline void raw_local_irq_disable
static inline void raw_local_irq_enable(void)
{
#ifdef CONFIG_IPIPE
+ barrier();
__ipipe_unstall_root();
#else
__asm__ __volatile__("sti" : : : "memory");
@@ -96,17 +100,29 @@ static inline void halt(void)
*/
static inline unsigned long __raw_local_irq_save(void)
{
+#ifdef CONFIG_IPIPE
+ unsigned long flags = (!__ipipe_test_and_stall_root()) << 9;
+
+ barrier();
+#else
unsigned long flags = __raw_local_save_flags();
raw_local_irq_disable();
-
+#endif
return flags;
}
#else
#ifdef CONFIG_IPIPE
-#define DISABLE_INTERRUPTS(clobbers) call __ipipe_stall_root ; sti
+#ifdef CONFIG_SMP
+#define DISABLE_INTERRUPTS(clobbers) call __ipipe_stall_root; sti
+#else /* !CONFIG_SMP */
+/*
+ * Disable IRQs == set IPIPE_STALL_FLAG in ipipe_root.cpudata[0].status
+ */
+#define DISABLE_INTERRUPTS(clobbers) btsl $0,ipipe_root; sti
+#endif /* !CONFIG_SMP */
#define ENABLE_INTERRUPTS(clobbers) call __ipipe_unstall_root
#define ENABLE_INTERRUPTS_HW_COND sti
#define DISABLE_INTERRUPTS_HW(clobbers) cli
---
include/asm-x86_64/irqflags.h | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
Index: linux-2.6.20-x64/include/asm-x86_64/irqflags.h
===================================================================
--- linux-2.6.20-x64.orig/include/asm-x86_64/irqflags.h
+++ linux-2.6.20-x64/include/asm-x86_64/irqflags.h
@@ -24,6 +24,7 @@ static inline unsigned long __raw_local_
#ifdef CONFIG_IPIPE
flags = (!__ipipe_test_root()) << 9;
+ barrier();
#else
__asm__ __volatile__(
"# __raw_save_flags\n\t"
@@ -43,6 +44,7 @@ static inline unsigned long __raw_local_
static inline void raw_local_irq_restore(unsigned long flags)
{
#ifdef CONFIG_IPIPE
+ barrier();
__ipipe_restore_root(!(flags & 0x200));
#else
__asm__ __volatile__(
@@ -85,6 +87,7 @@ static inline void raw_local_irq_disable
{
#ifdef CONFIG_IPIPE
__ipipe_stall_root();
+ barrier();
#else
__asm__ __volatile__("cli" : : : "memory");
#endif
@@ -93,6 +96,7 @@ static inline void raw_local_irq_disable
static inline void raw_local_irq_enable(void)
{
#ifdef CONFIG_IPIPE
+ barrier();
__ipipe_unstall_root();
#else
__asm__ __volatile__("sti" : : : "memory");
@@ -112,10 +116,15 @@ static inline int raw_irqs_disabled_flag
static inline unsigned long __raw_local_irq_save(void)
{
+#ifdef CONFIG_IPIPE
+ unsigned long flags = (!__ipipe_test_and_stall_root()) << 9;
+
+ barrier();
+#else
unsigned long flags = __raw_local_save_flags();
raw_local_irq_disable();
-
+#endif
return flags;
}
signature.asc
Description: OpenPGP digital signature
_______________________________________________ Adeos-main mailing list [email protected] https://mail.gna.org/listinfo/adeos-main
