Commit-ID: 0b95364f977c180e1f336e00273fda5d3eca54b4
Gitweb: http://git.kernel.org/tip/0b95364f977c180e1f336e00273fda5d3eca54b4
Author: Paolo Bonzini <pbonz...@redhat.com>
AuthorDate: Mon, 20 Jun 2016 16:58:29 +0200
Committer: Ingo Molnar <mi...@kernel.org>
CommitDate: Sat, 9 Jul 2016 10:44:01 +0200
x86/entry: Avoid interrupt flag save and restore
Thanks to all the work that was done by Andy Lutomirski and others,
enter_from_user_mode() and prepare_exit_to_usermode() are now called only with
interrupts disabled. Let's provide them a version of user_enter()/user_exit()
that skips saving and restoring the interrupt flag.
On an AMD-based machine I tested this patch on, with force-enabled
context tracking, the speed-up in system calls was 90 clock cycles or 6%,
measured with the following simple benchmark:
#include <sys/signal.h>
#include <time.h>
#include <unistd.h>
#include <stdio.h>
unsigned long rdtsc()
{
unsigned long result;
asm volatile("rdtsc; shl $32, %%rdx; mov %%eax, %%eax\n"
"or %%rdx, %%rax" : "=a" (result) : : "rdx");
return result;
}
int main()
{
unsigned long tsc1, tsc2;
int pid = getpid();
int i;
tsc1 = rdtsc();
for (i = 0; i < 100000000; i++)
kill(pid, SIGWINCH);
tsc2 = rdtsc();
printf("%ld\n", tsc2 - tsc1);
}
Signed-off-by: Paolo Bonzini <pbonz...@redhat.com>
Reviewed-by: Rik van Riel <r...@redhat.com>
Reviewed-by: Andy Lutomirski <l...@kernel.org>
Reviewed-by: Rik van Riel <r...@redhat.com>
Reviewed-by: Andy Lutomirski <l...@kernel.org>
Reviewed-by: Rik van Riel <r...@redhat.com>
Reviewed-by: Andy Lutomirski <l...@kernel.org>
Reviewed-by: Rik van Riel <r...@redhat.com>
Reviewed-by: Andy Lutomirski <l...@kernel.org>
Acked-by: Paolo Bonzini <pbonz...@redhat.com>
Cc: Borislav Petkov <b...@alien8.de>
Cc: Brian Gerst <brge...@gmail.com>
Cc: Denys Vlasenko <dvlas...@redhat.com>
Cc: H. Peter Anvin <h...@zytor.com>
Cc: Josh Poimboeuf <jpoim...@redhat.com>
Cc: Linus Torvalds <torva...@linux-foundation.org>
Cc: Peter Zijlstra <pet...@infradead.org>
Cc: Thomas Gleixner <t...@linutronix.de>
Cc: k...@vger.kernel.org
Link:
http://lkml.kernel.org/r/1466434712-31440-2-git-send-email-pbonz...@redhat.com
Signed-off-by: Ingo Molnar <mi...@kernel.org>
---
arch/x86/entry/common.c | 4 ++--
include/linux/context_tracking.h | 15 +++++++++++++++
2 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index ec138e5..618bc61 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -43,7 +43,7 @@ static struct thread_info *pt_regs_to_thread_info(struct
pt_regs *regs)
__visible void enter_from_user_mode(void)
{
CT_WARN_ON(ct_state() != CONTEXT_USER);
- user_exit();
+ user_exit_irqoff();
}
#else
static inline void enter_from_user_mode(void) {}
@@ -274,7 +274,7 @@ __visible inline void prepare_exit_to_usermode(struct
pt_regs *regs)
ti->status &= ~TS_COMPAT;
#endif
- user_enter();
+ user_enter_irqoff();
}
#define SYSCALL_EXIT_WORK_FLAGS \
diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h
index d259274..d9aef2a 100644
--- a/include/linux/context_tracking.h
+++ b/include/linux/context_tracking.h
@@ -31,6 +31,19 @@ static inline void user_exit(void)
context_tracking_exit(CONTEXT_USER);
}
+/* Called with interrupts disabled. */
+static inline void user_enter_irqoff(void)
+{
+ if (context_tracking_is_enabled())
+ __context_tracking_enter(CONTEXT_USER);
+
+}
+static inline void user_exit_irqoff(void)
+{
+ if (context_tracking_is_enabled())
+ __context_tracking_exit(CONTEXT_USER);
+}
+
static inline enum ctx_state exception_enter(void)
{
enum ctx_state prev_ctx;
@@ -69,6 +82,8 @@ static inline enum ctx_state ct_state(void)
#else
static inline void user_enter(void) { }
static inline void user_exit(void) { }
+static inline void user_enter_irqoff(void) { }
+static inline void user_exit_irqoff(void) { }
static inline enum ctx_state exception_enter(void) { return 0; }
static inline void exception_exit(enum ctx_state prev_ctx) { }
static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; }
