v3->v4
- Addressd as per review feedback for some editorial comments.
v3: https://lore.kernel.org/all/[email protected]/
---
arch/powerpc/include/asm/uaccess.h | 65 +++++++++++++++++++++---------
arch/powerpc/lib/copyuser_64.S | 1 +
arch/powerpc/lib/copyuser_power7.S | 45 +++++++--------------
arch/powerpc/lib/vmx-helper.c | 2 +
4 files changed, 63 insertions(+), 50 deletions(-)
diff --git a/arch/powerpc/include/asm/uaccess.h
b/arch/powerpc/include/asm/uaccess.h
index ba1d878c3f40..8b25be472cf0 100644
--- a/arch/powerpc/include/asm/uaccess.h
+++ b/arch/powerpc/include/asm/uaccess.h
@@ -15,6 +15,9 @@
#define TASK_SIZE_MAX TASK_SIZE_USER64
#endif
+/* Threshold above which VMX copy path is used */
+#define VMX_COPY_THRESHOLD 3328
+
#include <asm-generic/access_ok.h>
/*
@@ -326,40 +329,62 @@ do {
\
extern unsigned long __copy_tofrom_user(void __user *to,
const void __user *from, unsigned long size);
-#ifdef __powerpc64__
-static inline unsigned long
-raw_copy_in_user(void __user *to, const void __user *from, unsigned long n)
+unsigned long __copy_tofrom_user_base(void __user *to,
+ const void __user *from, unsigned long
size);
+
+unsigned long __copy_tofrom_user_power7_vmx(void __user *to,
+ const void __user *from, unsigned
long size);
+
+static __always_inline bool will_use_vmx(unsigned long n)
+{
+ return IS_ENABLED(CONFIG_ALTIVEC) && cpu_has_feature(CPU_FTR_VMX_COPY)
&&
+ n > VMX_COPY_THRESHOLD;
+}
+
+static __always_inline unsigned long
+raw_copy_tofrom_user(void __user *to, const void __user *from,
+ unsigned long n, unsigned long dir)
{
unsigned long ret;
- barrier_nospec();
- allow_user_access(to, KUAP_READ_WRITE);
+ if (will_use_vmx(n) && enter_vmx_usercopy()) {
+ allow_user_access(to, dir);
+ ret = __copy_tofrom_user_power7_vmx(to, from, n);
+ prevent_user_access(dir);
+ exit_vmx_usercopy();
+
+ if (unlikely(ret)) {
+ allow_user_access(to, dir);
+ ret = __copy_tofrom_user_base(to, from, n);
+ prevent_user_access(dir);
+ }
+ return ret;
+ }
+
+ allow_user_access(to, dir);
ret = __copy_tofrom_user(to, from, n);
- prevent_user_access(KUAP_READ_WRITE);
+ prevent_user_access(dir);
return ret;
}
-#endif /* __powerpc64__ */
-static inline unsigned long raw_copy_from_user(void *to,
- const void __user *from, unsigned long n)
+#ifdef CONFIG_PPC64
+static inline unsigned long
+raw_copy_in_user(void __user *to, const void __user *from, unsigned long n)
{
- unsigned long ret;
+ barrier_nospec();
+ return raw_copy_tofrom_user(to, from, n, KUAP_READ_WRITE);
+}
+#endif /* CONFIG_PPC64 */
- allow_user_access(NULL, KUAP_READ);
- ret = __copy_tofrom_user((__force void __user *)to, from, n);
- prevent_user_access(KUAP_READ);
- return ret;
+static inline unsigned long raw_copy_from_user(void *to, const void __user
*from, unsigned long n)
+{
+ return raw_copy_tofrom_user((__force void __user *)to, from, n,
KUAP_READ);
}
static inline unsigned long
raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
- unsigned long ret;
-
- allow_user_access(to, KUAP_WRITE);
- ret = __copy_tofrom_user(to, (__force const void __user *)from, n);
- prevent_user_access(KUAP_WRITE);
- return ret;
+ return raw_copy_tofrom_user(to, (__force const void __user *)from, n,
KUAP_WRITE);
}
unsigned long __arch_clear_user(void __user *addr, unsigned long size);
diff --git a/arch/powerpc/lib/copyuser_64.S b/arch/powerpc/lib/copyuser_64.S
index 9af969d2cc0c..25a99108caff 100644
--- a/arch/powerpc/lib/copyuser_64.S
+++ b/arch/powerpc/lib/copyuser_64.S
@@ -562,3 +562,4 @@ exc; std r10,32(3)
li r5,4096
b .Ldst_aligned
EXPORT_SYMBOL(__copy_tofrom_user)
+EXPORT_SYMBOL(__copy_tofrom_user_base)
diff --git a/arch/powerpc/lib/copyuser_power7.S
b/arch/powerpc/lib/copyuser_power7.S
index 8474c682a178..17dbcfbae25f 100644
--- a/arch/powerpc/lib/copyuser_power7.S
+++ b/arch/powerpc/lib/copyuser_power7.S
@@ -5,13 +5,9 @@
*
* Author: Anton Blanchard <[email protected]>
*/
+#include <linux/export.h>
#include <asm/ppc_asm.h>
-#ifndef SELFTEST_CASE
-/* 0 == don't use VMX, 1 == use VMX */
-#define SELFTEST_CASE 0
-#endif
-
#ifdef __BIG_ENDIAN__
#define LVS(VRT,RA,RB) lvsl VRT,RA,RB
#define VPERM(VRT,VRA,VRB,VRC) vperm VRT,VRA,VRB,VRC
@@ -47,10 +43,14 @@
ld r15,STK_REG(R15)(r1)
ld r14,STK_REG(R14)(r1)
.Ldo_err3:
- bl CFUNC(exit_vmx_usercopy)
+ ld r6,STK_REG(R31)(r1) /* original destination pointer */
+ ld r5,STK_REG(R29)(r1) /* original number of bytes */
+ subf r7,r6,r3 /* #bytes copied */
+ subf r3,r7,r5 /* #bytes not copied in r3 */
ld r0,STACKFRAMESIZE+16(r1)
mtlr r0
- b .Lexit
+ addi r1,r1,STACKFRAMESIZE
+ blr
#endif /* CONFIG_ALTIVEC */
.Ldo_err2:
@@ -74,7 +74,6 @@
_GLOBAL(__copy_tofrom_user_power7)
cmpldi r5,16
- cmpldi cr1,r5,3328
std r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
std r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
@@ -82,12 +81,6 @@ _GLOBAL(__copy_tofrom_user_power7)
blt .Lshort_copy
-#ifdef CONFIG_ALTIVEC
-test_feature = SELFTEST_CASE
-BEGIN_FTR_SECTION
- bgt cr1,.Lvmx_copy
-END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
-#endif
.Lnonvmx_copy:
/* Get the source 8B aligned */
@@ -263,23 +256,14 @@ err1; stb r0,0(r3)
15: li r3,0
blr
-.Lunwind_stack_nonvmx_copy:
- addi r1,r1,STACKFRAMESIZE
- b .Lnonvmx_copy
-
-.Lvmx_copy:
#ifdef CONFIG_ALTIVEC
+_GLOBAL(__copy_tofrom_user_power7_vmx)
mflr r0
std r0,16(r1)
stdu r1,-STACKFRAMESIZE(r1)
- bl CFUNC(enter_vmx_usercopy)
- cmpwi cr1,r3,0
- ld r0,STACKFRAMESIZE+16(r1)
- ld r3,STK_REG(R31)(r1)
- ld r4,STK_REG(R30)(r1)
- ld r5,STK_REG(R29)(r1)
- mtlr r0
+ std r3,STK_REG(R31)(r1)
+ std r5,STK_REG(R29)(r1)
/*
* We prefetch both the source and destination using enhanced touch
* instructions. We use a stream ID of 0 for the load side and
@@ -300,8 +284,6 @@ err1; stb r0,0(r3)
DCBT_SETUP_STREAMS(r6, r7, r9, r10, r8)
- beq cr1,.Lunwind_stack_nonvmx_copy
-
/*
* If source and destination are not relatively aligned we use a
* slower permute loop.
@@ -478,7 +460,8 @@ err3; lbz r0,0(r4)
err3; stb r0,0(r3)
15: addi r1,r1,STACKFRAMESIZE
- b CFUNC(exit_vmx_usercopy) /* tail call optimise */
+ li r3,0
+ blr
.Lvmx_unaligned_copy:
/* Get the destination 16B aligned */
@@ -681,5 +664,7 @@ err3; lbz r0,0(r4)
err3; stb r0,0(r3)
15: addi r1,r1,STACKFRAMESIZE
- b CFUNC(exit_vmx_usercopy) /* tail call optimise */
+ li r3,0
+ blr
+EXPORT_SYMBOL(__copy_tofrom_user_power7_vmx)
#endif /* CONFIG_ALTIVEC */
diff --git a/arch/powerpc/lib/vmx-helper.c b/arch/powerpc/lib/vmx-helper.c
index 54340912398f..554b248002b4 100644
--- a/arch/powerpc/lib/vmx-helper.c
+++ b/arch/powerpc/lib/vmx-helper.c
@@ -27,6 +27,7 @@ int enter_vmx_usercopy(void)
return 1;
}
+EXPORT_SYMBOL(enter_vmx_usercopy);
/*
* This function must return 0 because we tail call optimise when calling
@@ -49,6 +50,7 @@ int exit_vmx_usercopy(void)
set_dec(1);
return 0;
}
+EXPORT_SYMBOL(exit_vmx_usercopy);
int enter_vmx_ops(void)
{
