[tip:x86/asm] x86/entry: Avoid interrupt flag save and restore

[tip-bot for Paolo Bonzini]

Commit-ID:  2e9d1e150abf88cb63e5d34ca286edbb95b4c53d
Gitweb: http://git.kernel.org/tip/2e9d1e150abf88cb63e5d34ca286edbb95b4c53d
Author: Paolo Bonzini 
AuthorDate: Mon, 20 Jun 2016 16:58:29 +0200
Committer:  Ingo Molnar 
CommitDate: Sun, 10 Jul 2016 13:33:02 +0200

x86/entry: Avoid interrupt flag save and restore

Thanks to all the work that was done by Andy Lutomirski and others,
enter_from_user_mode() and prepare_exit_to_usermode() are now called only with
interrupts disabled.  Let's provide them a version of user_enter()/user_exit()
that skips saving and restoring the interrupt flag.

On an AMD-based machine I tested this patch on, with force-enabled
context tracking, the speed-up in system calls was 90 clock cycles or 6%,
measured with the following simple benchmark:

#include 
#include 
#include 
#include 

unsigned long rdtsc()
{
unsigned long result;
asm volatile("rdtsc; shl $32, %%rdx; mov %%eax, %%eax\n"
 "or %%rdx, %%rax" : "=a" (result) : : "rdx");
return result;
}

int main()
{
unsigned long tsc1, tsc2;
int pid = getpid();
int i;

tsc1 = rdtsc();
for (i = 0; i < 1; i++)
kill(pid, SIGWINCH);
tsc2 = rdtsc();

printf("%ld\n", tsc2 - tsc1);
}

Signed-off-by: Paolo Bonzini 
Reviewed-by: Rik van Riel 
Reviewed-by: Andy Lutomirski 
Acked-by: Paolo Bonzini 
Cc: Borislav Petkov 
Cc: Brian Gerst 
Cc: Denys Vlasenko 
Cc: H. Peter Anvin 
Cc: Josh Poimboeuf 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: k...@vger.kernel.org
Link: 
http://lkml.kernel.org/r/1466434712-31440-2-git-send-email-pbonz...@redhat.com
Signed-off-by: Ingo Molnar 
---
 arch/x86/entry/common.c  |  4 ++--
 include/linux/context_tracking.h | 15 +++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index ec138e5..618bc61 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -43,7 +43,7 @@ static struct thread_info *pt_regs_to_thread_info(struct 
pt_regs *regs)
 __visible void enter_from_user_mode(void)
 {
CT_WARN_ON(ct_state() != CONTEXT_USER);
-   user_exit();
+   user_exit_irqoff();
 }
 #else
 static inline void enter_from_user_mode(void) {}
@@ -274,7 +274,7 @@ __visible inline void prepare_exit_to_usermode(struct 
pt_regs *regs)
ti->status &= ~TS_COMPAT;
 #endif
 
-   user_enter();
+   user_enter_irqoff();
 }
 
 #define SYSCALL_EXIT_WORK_FLAGS\
diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h
index d259274..d9aef2a 100644
--- a/include/linux/context_tracking.h
+++ b/include/linux/context_tracking.h
@@ -31,6 +31,19 @@ static inline void user_exit(void)
context_tracking_exit(CONTEXT_USER);
 }
 
+/* Called with interrupts disabled.  */
+static inline void user_enter_irqoff(void)
+{
+   if (context_tracking_is_enabled())
+   __context_tracking_enter(CONTEXT_USER);
+
+}
+static inline void user_exit_irqoff(void)
+{
+   if (context_tracking_is_enabled())
+   __context_tracking_exit(CONTEXT_USER);
+}
+
 static inline enum ctx_state exception_enter(void)
 {
enum ctx_state prev_ctx;
@@ -69,6 +82,8 @@ static inline enum ctx_state ct_state(void)
 #else
 static inline void user_enter(void) { }
 static inline void user_exit(void) { }
+static inline void user_enter_irqoff(void) { }
+static inline void user_exit_irqoff(void) { }
 static inline enum ctx_state exception_enter(void) { return 0; }
 static inline void exception_exit(enum ctx_state prev_ctx) { }
 static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; }

[tip:x86/asm] x86/entry: Avoid interrupt flag save and restore

[tip-bot for Paolo Bonzini]

Commit-ID:  2e9d1e150abf88cb63e5d34ca286edbb95b4c53d
Gitweb: http://git.kernel.org/tip/2e9d1e150abf88cb63e5d34ca286edbb95b4c53d
Author: Paolo Bonzini 
AuthorDate: Mon, 20 Jun 2016 16:58:29 +0200
Committer:  Ingo Molnar 
CommitDate: Sun, 10 Jul 2016 13:33:02 +0200

x86/entry: Avoid interrupt flag save and restore

Thanks to all the work that was done by Andy Lutomirski and others,
enter_from_user_mode() and prepare_exit_to_usermode() are now called only with
interrupts disabled.  Let's provide them a version of user_enter()/user_exit()
that skips saving and restoring the interrupt flag.

On an AMD-based machine I tested this patch on, with force-enabled
context tracking, the speed-up in system calls was 90 clock cycles or 6%,
measured with the following simple benchmark:

#include 
#include 
#include 
#include 

unsigned long rdtsc()
{
unsigned long result;
asm volatile("rdtsc; shl $32, %%rdx; mov %%eax, %%eax\n"
 "or %%rdx, %%rax" : "=a" (result) : : "rdx");
return result;
}

int main()
{
unsigned long tsc1, tsc2;
int pid = getpid();
int i;

tsc1 = rdtsc();
for (i = 0; i < 1; i++)
kill(pid, SIGWINCH);
tsc2 = rdtsc();

printf("%ld\n", tsc2 - tsc1);
}

Signed-off-by: Paolo Bonzini 
Reviewed-by: Rik van Riel 
Reviewed-by: Andy Lutomirski 
Acked-by: Paolo Bonzini 
Cc: Borislav Petkov 
Cc: Brian Gerst 
Cc: Denys Vlasenko 
Cc: H. Peter Anvin 
Cc: Josh Poimboeuf 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: k...@vger.kernel.org
Link: 
http://lkml.kernel.org/r/1466434712-31440-2-git-send-email-pbonz...@redhat.com
Signed-off-by: Ingo Molnar 
---
 arch/x86/entry/common.c  |  4 ++--
 include/linux/context_tracking.h | 15 +++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index ec138e5..618bc61 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -43,7 +43,7 @@ static struct thread_info *pt_regs_to_thread_info(struct 
pt_regs *regs)
 __visible void enter_from_user_mode(void)
 {
CT_WARN_ON(ct_state() != CONTEXT_USER);
-   user_exit();
+   user_exit_irqoff();
 }
 #else
 static inline void enter_from_user_mode(void) {}
@@ -274,7 +274,7 @@ __visible inline void prepare_exit_to_usermode(struct 
pt_regs *regs)
ti->status &= ~TS_COMPAT;
 #endif
 
-   user_enter();
+   user_enter_irqoff();
 }
 
 #define SYSCALL_EXIT_WORK_FLAGS\
diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h
index d259274..d9aef2a 100644
--- a/include/linux/context_tracking.h
+++ b/include/linux/context_tracking.h
@@ -31,6 +31,19 @@ static inline void user_exit(void)
context_tracking_exit(CONTEXT_USER);
 }
 
+/* Called with interrupts disabled.  */
+static inline void user_enter_irqoff(void)
+{
+   if (context_tracking_is_enabled())
+   __context_tracking_enter(CONTEXT_USER);
+
+}
+static inline void user_exit_irqoff(void)
+{
+   if (context_tracking_is_enabled())
+   __context_tracking_exit(CONTEXT_USER);
+}
+
 static inline enum ctx_state exception_enter(void)
 {
enum ctx_state prev_ctx;
@@ -69,6 +82,8 @@ static inline enum ctx_state ct_state(void)
 #else
 static inline void user_enter(void) { }
 static inline void user_exit(void) { }
+static inline void user_enter_irqoff(void) { }
+static inline void user_exit_irqoff(void) { }
 static inline enum ctx_state exception_enter(void) { return 0; }
 static inline void exception_exit(enum ctx_state prev_ctx) { }
 static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; }

[tip:x86/asm] x86/entry: Avoid interrupt flag save and restore

[tip-bot for Paolo Bonzini]

Commit-ID:  0b95364f977c180e1f336e00273fda5d3eca54b4
Gitweb: http://git.kernel.org/tip/0b95364f977c180e1f336e00273fda5d3eca54b4
Author: Paolo Bonzini 
AuthorDate: Mon, 20 Jun 2016 16:58:29 +0200
Committer:  Ingo Molnar 
CommitDate: Sat, 9 Jul 2016 10:44:01 +0200

x86/entry: Avoid interrupt flag save and restore

Thanks to all the work that was done by Andy Lutomirski and others,
enter_from_user_mode() and prepare_exit_to_usermode() are now called only with
interrupts disabled.  Let's provide them a version of user_enter()/user_exit()
that skips saving and restoring the interrupt flag.

On an AMD-based machine I tested this patch on, with force-enabled
context tracking, the speed-up in system calls was 90 clock cycles or 6%,
measured with the following simple benchmark:

#include 
#include 
#include 
#include 

unsigned long rdtsc()
{
unsigned long result;
asm volatile("rdtsc; shl $32, %%rdx; mov %%eax, %%eax\n"
 "or %%rdx, %%rax" : "=a" (result) : : "rdx");
return result;
}

int main()
{
unsigned long tsc1, tsc2;
int pid = getpid();
int i;

tsc1 = rdtsc();
for (i = 0; i < 1; i++)
kill(pid, SIGWINCH);
tsc2 = rdtsc();

printf("%ld\n", tsc2 - tsc1);
}

Signed-off-by: Paolo Bonzini 
Reviewed-by: Rik van Riel 
Reviewed-by: Andy Lutomirski 
Reviewed-by: Rik van Riel 
Reviewed-by: Andy Lutomirski 
Reviewed-by: Rik van Riel 
Reviewed-by: Andy Lutomirski 
Reviewed-by: Rik van Riel 
Reviewed-by: Andy Lutomirski 
Acked-by: Paolo Bonzini 
Cc: Borislav Petkov 
Cc: Brian Gerst 
Cc: Denys Vlasenko 
Cc: H. Peter Anvin 
Cc: Josh Poimboeuf 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: k...@vger.kernel.org
Link: 
http://lkml.kernel.org/r/1466434712-31440-2-git-send-email-pbonz...@redhat.com
Signed-off-by: Ingo Molnar 
---
 arch/x86/entry/common.c  |  4 ++--
 include/linux/context_tracking.h | 15 +++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index ec138e5..618bc61 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -43,7 +43,7 @@ static struct thread_info *pt_regs_to_thread_info(struct 
pt_regs *regs)
 __visible void enter_from_user_mode(void)
 {
CT_WARN_ON(ct_state() != CONTEXT_USER);
-   user_exit();
+   user_exit_irqoff();
 }
 #else
 static inline void enter_from_user_mode(void) {}
@@ -274,7 +274,7 @@ __visible inline void prepare_exit_to_usermode(struct 
pt_regs *regs)
ti->status &= ~TS_COMPAT;
 #endif
 
-   user_enter();
+   user_enter_irqoff();
 }
 
 #define SYSCALL_EXIT_WORK_FLAGS\
diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h
index d259274..d9aef2a 100644
--- a/include/linux/context_tracking.h
+++ b/include/linux/context_tracking.h
@@ -31,6 +31,19 @@ static inline void user_exit(void)
context_tracking_exit(CONTEXT_USER);
 }
 
+/* Called with interrupts disabled.  */
+static inline void user_enter_irqoff(void)
+{
+   if (context_tracking_is_enabled())
+   __context_tracking_enter(CONTEXT_USER);
+
+}
+static inline void user_exit_irqoff(void)
+{
+   if (context_tracking_is_enabled())
+   __context_tracking_exit(CONTEXT_USER);
+}
+
 static inline enum ctx_state exception_enter(void)
 {
enum ctx_state prev_ctx;
@@ -69,6 +82,8 @@ static inline enum ctx_state ct_state(void)
 #else
 static inline void user_enter(void) { }
 static inline void user_exit(void) { }
+static inline void user_enter_irqoff(void) { }
+static inline void user_exit_irqoff(void) { }
 static inline enum ctx_state exception_enter(void) { return 0; }
 static inline void exception_exit(enum ctx_state prev_ctx) { }
 static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; }

[tip:x86/asm] x86/entry: Avoid interrupt flag save and restore

[tip-bot for Paolo Bonzini]

Commit-ID:  0b95364f977c180e1f336e00273fda5d3eca54b4
Gitweb: http://git.kernel.org/tip/0b95364f977c180e1f336e00273fda5d3eca54b4
Author: Paolo Bonzini 
AuthorDate: Mon, 20 Jun 2016 16:58:29 +0200
Committer:  Ingo Molnar 
CommitDate: Sat, 9 Jul 2016 10:44:01 +0200

x86/entry: Avoid interrupt flag save and restore

Thanks to all the work that was done by Andy Lutomirski and others,
enter_from_user_mode() and prepare_exit_to_usermode() are now called only with
interrupts disabled.  Let's provide them a version of user_enter()/user_exit()
that skips saving and restoring the interrupt flag.

On an AMD-based machine I tested this patch on, with force-enabled
context tracking, the speed-up in system calls was 90 clock cycles or 6%,
measured with the following simple benchmark:

#include 
#include 
#include 
#include 

unsigned long rdtsc()
{
unsigned long result;
asm volatile("rdtsc; shl $32, %%rdx; mov %%eax, %%eax\n"
 "or %%rdx, %%rax" : "=a" (result) : : "rdx");
return result;
}

int main()
{
unsigned long tsc1, tsc2;
int pid = getpid();
int i;

tsc1 = rdtsc();
for (i = 0; i < 1; i++)
kill(pid, SIGWINCH);
tsc2 = rdtsc();

printf("%ld\n", tsc2 - tsc1);
}

Signed-off-by: Paolo Bonzini 
Reviewed-by: Rik van Riel 
Reviewed-by: Andy Lutomirski 
Reviewed-by: Rik van Riel 
Reviewed-by: Andy Lutomirski 
Reviewed-by: Rik van Riel 
Reviewed-by: Andy Lutomirski 
Reviewed-by: Rik van Riel 
Reviewed-by: Andy Lutomirski 
Acked-by: Paolo Bonzini 
Cc: Borislav Petkov 
Cc: Brian Gerst 
Cc: Denys Vlasenko 
Cc: H. Peter Anvin 
Cc: Josh Poimboeuf 
Cc: Linus Torvalds 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: k...@vger.kernel.org
Link: 
http://lkml.kernel.org/r/1466434712-31440-2-git-send-email-pbonz...@redhat.com
Signed-off-by: Ingo Molnar 
---
 arch/x86/entry/common.c  |  4 ++--
 include/linux/context_tracking.h | 15 +++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index ec138e5..618bc61 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -43,7 +43,7 @@ static struct thread_info *pt_regs_to_thread_info(struct 
pt_regs *regs)
 __visible void enter_from_user_mode(void)
 {
CT_WARN_ON(ct_state() != CONTEXT_USER);
-   user_exit();
+   user_exit_irqoff();
 }
 #else
 static inline void enter_from_user_mode(void) {}
@@ -274,7 +274,7 @@ __visible inline void prepare_exit_to_usermode(struct 
pt_regs *regs)
ti->status &= ~TS_COMPAT;
 #endif
 
-   user_enter();
+   user_enter_irqoff();
 }
 
 #define SYSCALL_EXIT_WORK_FLAGS\
diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h
index d259274..d9aef2a 100644
--- a/include/linux/context_tracking.h
+++ b/include/linux/context_tracking.h
@@ -31,6 +31,19 @@ static inline void user_exit(void)
context_tracking_exit(CONTEXT_USER);
 }
 
+/* Called with interrupts disabled.  */
+static inline void user_enter_irqoff(void)
+{
+   if (context_tracking_is_enabled())
+   __context_tracking_enter(CONTEXT_USER);
+
+}
+static inline void user_exit_irqoff(void)
+{
+   if (context_tracking_is_enabled())
+   __context_tracking_exit(CONTEXT_USER);
+}
+
 static inline enum ctx_state exception_enter(void)
 {
enum ctx_state prev_ctx;
@@ -69,6 +82,8 @@ static inline enum ctx_state ct_state(void)
 #else
 static inline void user_enter(void) { }
 static inline void user_exit(void) { }
+static inline void user_enter_irqoff(void) { }
+static inline void user_exit_irqoff(void) { }
 static inline enum ctx_state exception_enter(void) { return 0; }
 static inline void exception_exit(enum ctx_state prev_ctx) { }
 static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; }