Hi,
has anyone looked at the netbsd xscale-versions of bcopyin/bcopyout/kcopy?
this is from netbsd bcopyinout.S:
#if defined(__XSCALE__) || defined(_ARM_ARCH_6)
/*
* armv6 and v7 have pld and strd so they can use the xscale
* bcopyinout as well.
*/
#include "bcopyinout_xscale.S"
#else
untested diff below i just scavenged from one of my dead branches,
just incase someone has the time and motivation to run it through some
performance testing or w/e.
-Artturi
diff --git a/sys/arch/arm/arm/bcopyinout.S b/sys/arch/arm/arm/bcopyinout.S
index 9a7d11865c0..bc2e58d22b7 100644
--- a/sys/arch/arm/arm/bcopyinout.S
+++ b/sys/arch/arm/arm/bcopyinout.S
@@ -41,7 +41,7 @@
#include <machine/asm.h>
#include <arm/sysreg.h>
-#ifdef __XSCALE__
+#ifdef CPU_ARMv7
#include "bcopyinout_xscale.S"
#else
diff --git a/sys/arch/arm/arm/bcopyinout_xscale.S
b/sys/arch/arm/arm/bcopyinout_xscale.S
new file mode 100644
index 00000000000..2e740eb96c2
--- /dev/null
+++ b/sys/arch/arm/arm/bcopyinout_xscale.S
@@ -0,0 +1,1139 @@
+/* $OpenBSD$ */
+/* $NetBSD: bcopyinout_xscale.S,v 1.11 2013/12/01 02:54:33 joerg Exp $
*/
+
+/*
+ * Copyright 2003 Wasabi Systems, Inc.
+ * All rights reserved.
+ *
+ * Written by Steve C. Woodford for Wasabi Systems, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed for the NetBSD Project by
+ * Wasabi Systems, Inc.
+ * 4. The name of Wasabi Systems, Inc. may not be used to endorse
+ * or promote products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+ .text
+ .align 2
+
+/*
+ * r0 = user space address
+ * r1 = kernel space address
+ * r2 = length
+ *
+ * Copies bytes from user space to kernel space
+ */
+ENTRY(copyin)
+ cmp r2, #0x00
+#if /* XXX or <= 0 like below? */ 1
+ moveq r0, #0
+ moveq pc, lr
+#else
+ movle r0, #0x00
+ RETc(le) /* Bail early if length is <= 0 */
+#endif
+ push {r10-r11, lr}
+
+ /* Get curcpu from TPIDRPRW. */
+ mrc CP15_TPIDRPRW(r10)
+ ldr r10, [r10, #CI_CURPCB]
+
+ mov r3, #0x00
+ adr ip, .Lcopyin_fault
+ ldr r11, [r10, #PCB_ONFAULT]
+ str ip, [r10, #PCB_ONFAULT]
+ bl .Lcopyin_guts
+ str r11, [r10, #PCB_ONFAULT]
+ mov r0, #0x00
+ pop {r10-r11, pc}
+
+.Lcopyin_fault:
+ str r11, [r10, #PCB_ONFAULT]
+ cmp r3, #0x00
+ popgt {r4-r7} /* r3 > 0 Restore r4-r7 */
+ poplt {r4-r9} /* r3 < 0 Restore r4-r9 */
+ pop {r10-r11, pc}
+
+.Lcopyin_guts:
+ pld [r0]
+ /* Word-align the destination buffer */
+ ands ip, r1, #0x03 /* Already word aligned? */
+ beq .Lcopyin_wordaligned /* Yup */
+ rsb ip, ip, #0x04
+ cmp r2, ip /* Enough bytes left to align it? */
+ blt .Lcopyin_l4_2 /* Nope. Just copy bytewise */
+ sub r2, r2, ip
+ rsbs ip, ip, #0x03
+ addne pc, pc, ip, lsl #3
+ nop
+ ldrbt ip, [r0], #0x01
+ strb ip, [r1], #0x01
+ ldrbt ip, [r0], #0x01
+ strb ip, [r1], #0x01
+ ldrbt ip, [r0], #0x01
+ strb ip, [r1], #0x01
+ cmp r2, #0x00 /* All done? */
+ moveq pc, lr
+
+ /* Destination buffer is now word aligned */
+.Lcopyin_wordaligned:
+ ands ip, r0, #0x03 /* Is src also word-aligned? */
+ bne .Lcopyin_bad_align /* Nope. Things just got bad */
+ cmp r2, #0x08 /* Less than 8 bytes remaining? */
+ blt .Lcopyin_w_less_than8
+
+ /* Quad-align the destination buffer */
+ tst r1, #0x07 /* Already quad aligned? */
+ ldrtne ip, [r0], #0x04
+ push {r4-r9} /* Free up some registers */
+ mov r3, #-1 /* Signal restore r4-r9 */
+ tst r1, #0x07 /* XXX: bug work-around */
+ subne r2, r2, #0x04
+ strne ip, [r1], #0x04
+
+ /* Destination buffer quad aligned, source is word aligned */
+ subs r2, r2, #0x80
+ blt .Lcopyin_w_lessthan128
+
+ /* Copy 128 bytes at a time */
+.Lcopyin_w_loop128:
+ ldrt r4, [r0], #0x04 /* LD:00-03 */
+ ldrt r5, [r0], #0x04 /* LD:04-07 */
+ pld [r0, #0x18] /* Prefetch 0x20 */
+ ldrt r6, [r0], #0x04 /* LD:08-0b */
+ ldrt r7, [r0], #0x04 /* LD:0c-0f */
+ ldrt r8, [r0], #0x04 /* LD:10-13 */
+ ldrt r9, [r0], #0x04 /* LD:14-17 */
+ strd r4, r5, [r1], #0x08 /* ST:00-07 */
+ ldrt r4, [r0], #0x04 /* LD:18-1b */
+ ldrt r5, [r0], #0x04 /* LD:1c-1f */
+ strd r6, r7, [r1], #0x08 /* ST:08-0f */
+ ldrt r6, [r0], #0x04 /* LD:20-23 */
+ ldrt r7, [r0], #0x04 /* LD:24-27 */
+ pld [r0, #0x18] /* Prefetch 0x40 */
+ strd r8, r9, [r1], #0x08 /* ST:10-17 */
+ ldrt r8, [r0], #0x04 /* LD:28-2b */
+ ldrt r9, [r0], #0x04 /* LD:2c-2f */
+ strd r4, r5, [r1], #0x08 /* ST:18-1f */
+ ldrt r4, [r0], #0x04 /* LD:30-33 */
+ ldrt r5, [r0], #0x04 /* LD:34-37 */
+ strd r6, r7, [r1], #0x08 /* ST:20-27 */
+ ldrt r6, [r0], #0x04 /* LD:38-3b */
+ ldrt r7, [r0], #0x04 /* LD:3c-3f */
+ strd r8, r9, [r1], #0x08 /* ST:28-2f */
+ ldrt r8, [r0], #0x04 /* LD:40-43 */
+ ldrt r9, [r0], #0x04 /* LD:44-47 */
+ pld [r0, #0x18] /* Prefetch 0x60 */
+ strd r4, r5, [r1], #0x08 /* ST:30-37 */
+ ldrt r4, [r0], #0x04 /* LD:48-4b */
+ ldrt r5, [r0], #0x04 /* LD:4c-4f */
+ strd r6, r7, [r1], #0x08 /* ST:38-3f */
+ ldrt r6, [r0], #0x04 /* LD:50-53 */
+ ldrt r7, [r0], #0x04 /* LD:54-57 */
+ strd r8, r9, [r1], #0x08 /* ST:40-47 */
+ ldrt r8, [r0], #0x04 /* LD:58-5b */
+ ldrt r9, [r0], #0x04 /* LD:5c-5f */
+ strd r4, r5, [r1], #0x08 /* ST:48-4f */
+ ldrt r4, [r0], #0x04 /* LD:60-63 */
+ ldrt r5, [r0], #0x04 /* LD:64-67 */
+ pld [r0, #0x18] /* Prefetch 0x80 */
+ strd r6, r7, [r1], #0x08 /* ST:50-57 */
+ ldrt r6, [r0], #0x04 /* LD:68-6b */
+ ldrt r7, [r0], #0x04 /* LD:6c-6f */
+ strd r8, r9, [r1], #0x08 /* ST:58-5f */
+ ldrt r8, [r0], #0x04 /* LD:70-73 */
+ ldrt r9, [r0], #0x04 /* LD:74-77 */
+ strd r4, r5, [r1], #0x08 /* ST:60-67 */
+ ldrt r4, [r0], #0x04 /* LD:78-7b */
+ ldrt r5, [r0], #0x04 /* LD:7c-7f */
+ strd r6, r7, [r1], #0x08 /* ST:68-6f */
+ strd r8, r9, [r1], #0x08 /* ST:70-77 */
+ subs r2, r2, #0x80
+ strd r4, r5, [r1], #0x08 /* ST:78-7f */
+ bge .Lcopyin_w_loop128
+
+.Lcopyin_w_lessthan128:
+ adds r2, r2, #0x80 /* Adjust for extra sub */
+ popeq {r4-r9}
+ moveq pc, lr /* Return now if done */
+ subs r2, r2, #0x20
+ blt .Lcopyin_w_lessthan32
+
+ /* Copy 32 bytes at a time */
+.Lcopyin_w_loop32:
+ ldrt r4, [r0], #0x04
+ ldrt r5, [r0], #0x04
+ pld [r0, #0x18]
+ ldrt r6, [r0], #0x04
+ ldrt r7, [r0], #0x04
+ ldrt r8, [r0], #0x04
+ ldrt r9, [r0], #0x04
+ strd r4, r5, [r1], #0x08
+ ldrt r4, [r0], #0x04
+ ldrt r5, [r0], #0x04
+ strd r6, r7, [r1], #0x08
+ strd r8, r9, [r1], #0x08
+ subs r2, r2, #0x20
+ strd r4, r5, [r1], #0x08
+ bge .Lcopyin_w_loop32
+
+.Lcopyin_w_lessthan32:
+ adds r2, r2, #0x20 /* Adjust for extra sub */
+ popeq {r4-r9}
+ moveq pc, lr /* Return now if done */
+
+ and r4, r2, #0x18
+ rsb r5, r4, #0x18
+ subs r2, r2, r4
+ add pc, pc, r5, lsl #1
+ nop
+
+ /* At least 24 bytes remaining */
+ ldrt r4, [r0], #0x04
+ ldrt r5, [r0], #0x04
+ nop
+ strd r4, r5, [r1], #0x08
+
+ /* At least 16 bytes remaining */
+ ldrt r4, [r0], #0x04
+ ldrt r5, [r0], #0x04
+ nop
+ strd r4, r5, [r1], #0x08
+
+ /* At least 8 bytes remaining */
+ ldrt r4, [r0], #0x04
+ ldrt r5, [r0], #0x04
+ nop
+ strd r4, r5, [r1], #0x08
+
+ /* Less than 8 bytes remaining */
+ pop {r4-r9}
+ moveq pc, lr /* Return now if done */
+ mov r3, #0x00
+
+.Lcopyin_w_less_than8:
+ subs r2, r2, #0x04
+ ldrtge ip, [r0], #0x04
+ strge ip, [r1], #0x04
+ moveq pc, lr /* Return now if done */
+ addlt r2, r2, #0x04
+ ldrbt ip, [r0], #0x01
+ cmp r2, #0x02
+ ldrbtge r2, [r0], #0x01
+ strb ip, [r1], #0x01
+ ldrbtgt ip, [r0]
+ strbge r2, [r1], #0x01
+ strbgt ip, [r1]
+ RET
+
+/*
+ * At this point, it has not been possible to word align both buffers.
+ * The destination buffer (r1) is word aligned, but the source buffer
+ * (r0) is not.
+ */
+.Lcopyin_bad_align:
+ push {r4-r7}
+ mov r3, #0x01
+ bic r0, r0, #0x03
+ cmp ip, #2
+ ldrt ip, [r0], #0x04
+ bgt .Lcopyin_bad3
+ beq .Lcopyin_bad2
+ b .Lcopyin_bad1
+
+.Lcopyin_bad1_loop16:
+ mov r4, ip, lsr #8
+ ldrt r5, [r0], #0x04
+ pld [r0, #0x018]
+ ldrt r6, [r0], #0x04
+ ldrt r7, [r0], #0x04
+ ldrt ip, [r0], #0x04
+ orr r4, r4, r5, lsl #24
+ mov r5, r5, lsr #8
+ orr r5, r5, r6, lsl #24
+ mov r6, r6, lsr #8
+ orr r6, r6, r7, lsl #24
+ mov r7, r7, lsr #8
+ orr r7, r7, ip, lsl #24
+ str r4, [r1], #0x04
+ str r5, [r1], #0x04
+ str r6, [r1], #0x04
+ str r7, [r1], #0x04
+.Lcopyin_bad1:
+ subs r2, r2, #0x10
+ bge .Lcopyin_bad1_loop16
+
+ adds r2, r2, #0x10
+ popeq {r4-r7}
+ moveq pc, lr /* Return now if done */
+ subs r2, r2, #0x04
+ sublt r0, r0, #0x03
+ blt .Lcopyin_l4
+
+.Lcopyin_bad1_loop4:
+ mov r4, ip, lsr #8
+ ldrt ip, [r0], #0x04
+ subs r2, r2, #0x04
+ orr r4, r4, ip, lsl #24
+ str r4, [r1], #0x04
+ bge .Lcopyin_bad1_loop4
+ sub r0, r0, #0x03
+ b .Lcopyin_l4
+
+.Lcopyin_bad2_loop16:
+ mov r4, ip, lsr #16
+ ldrt r5, [r0], #0x04
+ pld [r0, #0x018]
+ ldrt r6, [r0], #0x04
+ ldrt r7, [r0], #0x04
+ ldrt ip, [r0], #0x04
+ orr r4, r4, r5, lsl #16
+ mov r5, r5, lsr #16
+ orr r5, r5, r6, lsl #16
+ mov r6, r6, lsr #16
+ orr r6, r6, r7, lsl #16
+ mov r7, r7, lsr #16
+ orr r7, r7, ip, lsl #16
+ str r4, [r1], #0x04
+ str r5, [r1], #0x04
+ str r6, [r1], #0x04
+ str r7, [r1], #0x04
+.Lcopyin_bad2:
+ subs r2, r2, #0x10
+ bge .Lcopyin_bad2_loop16
+
+ adds r2, r2, #0x10
+ popeq {r4-r7}
+ moveq pc, lr /* Return now if done */
+ subs r2, r2, #0x04
+ sublt r0, r0, #0x02
+ blt .Lcopyin_l4
+
+.Lcopyin_bad2_loop4:
+ mov r4, ip, lsr #16
+ ldrt ip, [r0], #0x04
+ subs r2, r2, #0x04
+ orr r4, r4, ip, lsl #16
+ str r4, [r1], #0x04
+ bge .Lcopyin_bad2_loop4
+ sub r0, r0, #0x02
+ b .Lcopyin_l4
+
+.Lcopyin_bad3_loop16:
+ mov r4, ip, lsr #24
+ ldrt r5, [r0], #0x04
+ pld [r0, #0x018]
+ ldrt r6, [r0], #0x04
+ ldrt r7, [r0], #0x04
+ ldrt ip, [r0], #0x04
+ orr r4, r4, r5, lsl #8
+ mov r5, r5, lsr #24
+ orr r5, r5, r6, lsl #8
+ mov r6, r6, lsr #24
+ orr r6, r6, r7, lsl #8
+ mov r7, r7, lsr #24
+ orr r7, r7, ip, lsl #8
+ str r4, [r1], #0x04
+ str r5, [r1], #0x04
+ str r6, [r1], #0x04
+ str r7, [r1], #0x04
+.Lcopyin_bad3:
+ subs r2, r2, #0x10
+ bge .Lcopyin_bad3_loop16
+
+ adds r2, r2, #0x10
+ popeq {r4-r7}
+ moveq pc, lr /* Return now if done */
+ subs r2, r2, #0x04
+ sublt r0, r0, #0x01
+ blt .Lcopyin_l4
+
+.Lcopyin_bad3_loop4:
+ mov r4, ip, lsr #24
+ ldrt ip, [r0], #0x04
+ subs r2, r2, #0x04
+ orr r4, r4, ip, lsl #8
+ str r4, [r1], #0x04
+ bge .Lcopyin_bad3_loop4
+ sub r0, r0, #0x01
+
+.Lcopyin_l4:
+ pop {r4-r7}
+ mov r3, #0x00
+ adds r2, r2, #0x04
+ moveq pc, lr
+.Lcopyin_l4_2:
+ rsbs r2, r2, #0x03
+ addne pc, pc, r2, lsl #3
+ nop
+ ldrbt ip, [r0], #0x01
+ strb ip, [r1], #0x01
+ ldrbt ip, [r0], #0x01
+ strb ip, [r1], #0x01
+ ldrbt ip, [r0]
+ strb ip, [r1]
+ RET
+END(copyin)
+
+
+/*
+ * r0 = kernel space address
+ * r1 = user space address
+ * r2 = length
+ *
+ * Copies bytes from kernel space to user space
+ */
+ENTRY(copyout)
+ cmp r2, #0x00
+#if /* XXX or <= 0 like below? */ 1
+ moveq r0, #0
+ moveq pc, lr
+#else
+ movle r0, #0x00
+ RETc(le) /* Bail early if length is <= 0 */
+#endif
+
+ push {r10-r11, lr}
+
+ /* Get curcpu from TPIDRPRW. */
+ mrc CP15_TPIDRPRW(r10)
+ ldr r10, [r10, #CI_CURPCB]
+
+ mov r3, #0x00
+ adr ip, .Lcopyout_fault
+ ldr r11, [r10, #PCB_ONFAULT]
+ str ip, [r10, #PCB_ONFAULT]
+ bl .Lcopyout_guts
+ str r11, [r10, #PCB_ONFAULT]
+ mov r0, #0x00
+ pop {r10-r11, pc}
+
+.Lcopyout_fault:
+ str r11, [r10, #PCB_ONFAULT]
+ cmp r3, #0x00
+ popgt {r4-r7} /* r3 > 0 Restore r4-r7 */
+ poplt {r4-r9} /* r3 < 0 Restore r4-r9 */
+ pop {r10-r11, pc}
+
+.Lcopyout_guts:
+ pld [r0]
+ /* Word-align the destination buffer */
+ ands ip, r1, #0x03 /* Already word aligned? */
+ beq .Lcopyout_wordaligned /* Yup */
+ rsb ip, ip, #0x04
+ cmp r2, ip /* Enough bytes left to align it? */
+ blt .Lcopyout_l4_2 /* Nope. Just copy bytewise */
+ sub r2, r2, ip
+ rsbs ip, ip, #0x03
+ addne pc, pc, ip, lsl #3
+ nop
+ ldrb ip, [r0], #0x01
+ strbt ip, [r1], #0x01
+ ldrb ip, [r0], #0x01
+ strbt ip, [r1], #0x01
+ ldrb ip, [r0], #0x01
+ strbt ip, [r1], #0x01
+ cmp r2, #0x00 /* All done? */
+ moveq pc, lr
+
+ /* Destination buffer is now word aligned */
+.Lcopyout_wordaligned:
+ ands ip, r0, #0x03 /* Is src also word-aligned? */
+ bne .Lcopyout_bad_align /* Nope. Things just got bad */
+ cmp r2, #0x08 /* Less than 8 bytes remaining? */
+ blt .Lcopyout_w_less_than8
+
+ /* Quad-align the destination buffer */
+ tst r1, #0x07 /* Already quad aligned? */
+ ldrne ip, [r0], #0x04
+ push {r4-r9} /* Free up some registers */
+ mov r3, #-1 /* Signal restore r4-r9 */
+ tst r1, #0x07 /* XXX: bug work-around */
+ subne r2, r2, #0x04
+ strtne ip, [r1], #0x04
+
+ /* Destination buffer quad aligned, source is word aligned */
+ subs r2, r2, #0x80
+ blt .Lcopyout_w_lessthan128
+
+ /* Copy 128 bytes at a time */
+.Lcopyout_w_loop128:
+ ldr r4, [r0], #0x04 /* LD:00-03 */
+ ldr r5, [r0], #0x04 /* LD:04-07 */
+ pld [r0, #0x18] /* Prefetch 0x20 */
+ ldr r6, [r0], #0x04 /* LD:08-0b */
+ ldr r7, [r0], #0x04 /* LD:0c-0f */
+ ldr r8, [r0], #0x04 /* LD:10-13 */
+ ldr r9, [r0], #0x04 /* LD:14-17 */
+ strt r4, [r1], #0x04 /* ST:00-03 */
+ strt r5, [r1], #0x04 /* ST:04-07 */
+ ldr r4, [r0], #0x04 /* LD:18-1b */
+ ldr r5, [r0], #0x04 /* LD:1c-1f */
+ strt r6, [r1], #0x04 /* ST:08-0b */
+ strt r7, [r1], #0x04 /* ST:0c-0f */
+ ldr r6, [r0], #0x04 /* LD:20-23 */
+ ldr r7, [r0], #0x04 /* LD:24-27 */
+ pld [r0, #0x18] /* Prefetch 0x40 */
+ strt r8, [r1], #0x04 /* ST:10-13 */
+ strt r9, [r1], #0x04 /* ST:14-17 */
+ ldr r8, [r0], #0x04 /* LD:28-2b */
+ ldr r9, [r0], #0x04 /* LD:2c-2f */
+ strt r4, [r1], #0x04 /* ST:18-1b */
+ strt r5, [r1], #0x04 /* ST:1c-1f */
+ ldr r4, [r0], #0x04 /* LD:30-33 */
+ ldr r5, [r0], #0x04 /* LD:34-37 */
+ strt r6, [r1], #0x04 /* ST:20-23 */
+ strt r7, [r1], #0x04 /* ST:24-27 */
+ ldr r6, [r0], #0x04 /* LD:38-3b */
+ ldr r7, [r0], #0x04 /* LD:3c-3f */
+ strt r8, [r1], #0x04 /* ST:28-2b */
+ strt r9, [r1], #0x04 /* ST:2c-2f */
+ ldr r8, [r0], #0x04 /* LD:40-43 */
+ ldr r9, [r0], #0x04 /* LD:44-47 */
+ pld [r0, #0x18] /* Prefetch 0x60 */
+ strt r4, [r1], #0x04 /* ST:30-33 */
+ strt r5, [r1], #0x04 /* ST:34-37 */
+ ldr r4, [r0], #0x04 /* LD:48-4b */
+ ldr r5, [r0], #0x04 /* LD:4c-4f */
+ strt r6, [r1], #0x04 /* ST:38-3b */
+ strt r7, [r1], #0x04 /* ST:3c-3f */
+ ldr r6, [r0], #0x04 /* LD:50-53 */
+ ldr r7, [r0], #0x04 /* LD:54-57 */
+ strt r8, [r1], #0x04 /* ST:40-43 */
+ strt r9, [r1], #0x04 /* ST:44-47 */
+ ldr r8, [r0], #0x04 /* LD:58-5b */
+ ldr r9, [r0], #0x04 /* LD:5c-5f */
+ strt r4, [r1], #0x04 /* ST:48-4b */
+ strt r5, [r1], #0x04 /* ST:4c-4f */
+ ldr r4, [r0], #0x04 /* LD:60-63 */
+ ldr r5, [r0], #0x04 /* LD:64-67 */
+ pld [r0, #0x18] /* Prefetch 0x80 */
+ strt r6, [r1], #0x04 /* ST:50-53 */
+ strt r7, [r1], #0x04 /* ST:54-57 */
+ ldr r6, [r0], #0x04 /* LD:68-6b */
+ ldr r7, [r0], #0x04 /* LD:6c-6f */
+ strt r8, [r1], #0x04 /* ST:58-5b */
+ strt r9, [r1], #0x04 /* ST:5c-5f */
+ ldr r8, [r0], #0x04 /* LD:70-73 */
+ ldr r9, [r0], #0x04 /* LD:74-77 */
+ strt r4, [r1], #0x04 /* ST:60-63 */
+ strt r5, [r1], #0x04 /* ST:64-67 */
+ ldr r4, [r0], #0x04 /* LD:78-7b */
+ ldr r5, [r0], #0x04 /* LD:7c-7f */
+ strt r6, [r1], #0x04 /* ST:68-6b */
+ strt r7, [r1], #0x04 /* ST:6c-6f */
+ strt r8, [r1], #0x04 /* ST:70-73 */
+ strt r9, [r1], #0x04 /* ST:74-77 */
+ subs r2, r2, #0x80
+ strt r4, [r1], #0x04 /* ST:78-7b */
+ strt r5, [r1], #0x04 /* ST:7c-7f */
+ bge .Lcopyout_w_loop128
+
+.Lcopyout_w_lessthan128:
+ adds r2, r2, #0x80 /* Adjust for extra sub */
+ popeq {r4-r9}
+ moveq pc, lr /* Return now if done */
+ subs r2, r2, #0x20
+ blt .Lcopyout_w_lessthan32
+
+ /* Copy 32 bytes at a time */
+.Lcopyout_w_loop32:
+ ldr r4, [r0], #0x04
+ ldr r5, [r0], #0x04
+ pld [r0, #0x18]
+ ldr r6, [r0], #0x04
+ ldr r7, [r0], #0x04
+ ldr r8, [r0], #0x04
+ ldr r9, [r0], #0x04
+ strt r4, [r1], #0x04
+ strt r5, [r1], #0x04
+ ldr r4, [r0], #0x04
+ ldr r5, [r0], #0x04
+ strt r6, [r1], #0x04
+ strt r7, [r1], #0x04
+ strt r8, [r1], #0x04
+ strt r9, [r1], #0x04
+ subs r2, r2, #0x20
+ strt r4, [r1], #0x04
+ strt r5, [r1], #0x04
+ bge .Lcopyout_w_loop32
+
+.Lcopyout_w_lessthan32:
+ adds r2, r2, #0x20 /* Adjust for extra sub */
+ popeq {r4-r9}
+ moveq pc, lr /* Return now if done */
+
+ and r4, r2, #0x18
+ rsb r5, r4, #0x18
+ subs r2, r2, r4
+ add pc, pc, r5, lsl #1
+ nop
+
+ /* At least 24 bytes remaining */
+ ldr r4, [r0], #0x04
+ ldr r5, [r0], #0x04
+ strt r4, [r1], #0x04
+ strt r5, [r1], #0x04
+
+ /* At least 16 bytes remaining */
+ ldr r4, [r0], #0x04
+ ldr r5, [r0], #0x04
+ strt r4, [r1], #0x04
+ strt r5, [r1], #0x04
+
+ /* At least 8 bytes remaining */
+ ldr r4, [r0], #0x04
+ ldr r5, [r0], #0x04
+ strt r4, [r1], #0x04
+ strt r5, [r1], #0x04
+
+ /* Less than 8 bytes remaining */
+ pop {r4-r9}
+ moveq pc, lr /* Return now if done */
+ mov r3, #0x00
+
+.Lcopyout_w_less_than8:
+ subs r2, r2, #0x04
+ ldrge ip, [r0], #0x04
+ strtge ip, [r1], #0x04
+ moveq pc, lr /* Return now if done */
+ addlt r2, r2, #0x04
+ ldrb ip, [r0], #0x01
+ cmp r2, #0x02
+ ldrbge r2, [r0], #0x01
+ strbt ip, [r1], #0x01
+ ldrbgt ip, [r0]
+ strbtge r2, [r1], #0x01
+ strbtgt ip, [r1]
+ RET
+
+/*
+ * At this point, it has not been possible to word align both buffers.
+ * The destination buffer (r1) is word aligned, but the source buffer
+ * (r0) is not.
+ */
+.Lcopyout_bad_align:
+ push {r4-r7}
+ mov r3, #0x01
+ bic r0, r0, #0x03
+ cmp ip, #2
+ ldr ip, [r0], #0x04
+ bgt .Lcopyout_bad3
+ beq .Lcopyout_bad2
+ b .Lcopyout_bad1
+
+.Lcopyout_bad1_loop16:
+ mov r4, ip, lsr #8
+ ldr r5, [r0], #0x04
+ pld [r0, #0x018]
+ ldr r6, [r0], #0x04
+ ldr r7, [r0], #0x04
+ ldr ip, [r0], #0x04
+ orr r4, r4, r5, lsl #24
+ mov r5, r5, lsr #8
+ orr r5, r5, r6, lsl #24
+ mov r6, r6, lsr #8
+ orr r6, r6, r7, lsl #24
+ mov r7, r7, lsr #8
+ orr r7, r7, ip, lsl #24
+ strt r4, [r1], #0x04
+ strt r5, [r1], #0x04
+ strt r6, [r1], #0x04
+ strt r7, [r1], #0x04
+.Lcopyout_bad1:
+ subs r2, r2, #0x10
+ bge .Lcopyout_bad1_loop16
+
+ adds r2, r2, #0x10
+ popeq {r4-r7}
+ moveq pc, lr /* Return now if done */
+ subs r2, r2, #0x04
+ sublt r0, r0, #0x03
+ blt .Lcopyout_l4
+
+.Lcopyout_bad1_loop4:
+ mov r4, ip, lsr #8
+ ldr ip, [r0], #0x04
+ subs r2, r2, #0x04
+ orr r4, r4, ip, lsl #24
+ strt r4, [r1], #0x04
+ bge .Lcopyout_bad1_loop4
+ sub r0, r0, #0x03
+ b .Lcopyout_l4
+
+.Lcopyout_bad2_loop16:
+ mov r4, ip, lsr #16
+ ldr r5, [r0], #0x04
+ pld [r0, #0x018]
+ ldr r6, [r0], #0x04
+ ldr r7, [r0], #0x04
+ ldr ip, [r0], #0x04
+ orr r4, r4, r5, lsl #16
+ mov r5, r5, lsr #16
+ orr r5, r5, r6, lsl #16
+ mov r6, r6, lsr #16
+ orr r6, r6, r7, lsl #16
+ mov r7, r7, lsr #16
+ orr r7, r7, ip, lsl #16
+ strt r4, [r1], #0x04
+ strt r5, [r1], #0x04
+ strt r6, [r1], #0x04
+ strt r7, [r1], #0x04
+.Lcopyout_bad2:
+ subs r2, r2, #0x10
+ bge .Lcopyout_bad2_loop16
+
+ adds r2, r2, #0x10
+ popeq {r4-r7}
+ moveq pc, lr /* Return now if done */
+ subs r2, r2, #0x04
+ sublt r0, r0, #0x02
+ blt .Lcopyout_l4
+
+.Lcopyout_bad2_loop4:
+ mov r4, ip, lsr #16
+ ldr ip, [r0], #0x04
+ subs r2, r2, #0x04
+ orr r4, r4, ip, lsl #16
+ strt r4, [r1], #0x04
+ bge .Lcopyout_bad2_loop4
+ sub r0, r0, #0x02
+ b .Lcopyout_l4
+
+.Lcopyout_bad3_loop16:
+ mov r4, ip, lsr #24
+ ldr r5, [r0], #0x04
+ pld [r0, #0x018]
+ ldr r6, [r0], #0x04
+ ldr r7, [r0], #0x04
+ ldr ip, [r0], #0x04
+ orr r4, r4, r5, lsl #8
+ mov r5, r5, lsr #24
+ orr r5, r5, r6, lsl #8
+ mov r6, r6, lsr #24
+ orr r6, r6, r7, lsl #8
+ mov r7, r7, lsr #24
+ orr r7, r7, ip, lsl #8
+ strt r4, [r1], #0x04
+ strt r5, [r1], #0x04
+ strt r6, [r1], #0x04
+ strt r7, [r1], #0x04
+.Lcopyout_bad3:
+ subs r2, r2, #0x10
+ bge .Lcopyout_bad3_loop16
+
+ adds r2, r2, #0x10
+ popeq {r4-r7}
+ moveq pc, lr /* Return now if done */
+ subs r2, r2, #0x04
+ sublt r0, r0, #0x01
+ blt .Lcopyout_l4
+
+.Lcopyout_bad3_loop4:
+ mov r4, ip, lsr #24
+ ldr ip, [r0], #0x04
+ subs r2, r2, #0x04
+ orr r4, r4, ip, lsl #8
+ strt r4, [r1], #0x04
+ bge .Lcopyout_bad3_loop4
+ sub r0, r0, #0x01
+
+.Lcopyout_l4:
+ pop {r4-r7}
+ mov r3, #0x00
+ adds r2, r2, #0x04
+ moveq pc, lr
+.Lcopyout_l4_2:
+ rsbs r2, r2, #0x03
+ addne pc, pc, r2, lsl #3
+ nop
+ ldrb ip, [r0], #0x01
+ strbt ip, [r1], #0x01
+ ldrb ip, [r0], #0x01
+ strbt ip, [r1], #0x01
+ ldrb ip, [r0]
+ strbt ip, [r1]
+ RET
+END(copyout)
+
+/*
+ * r0 = kernel space source address
+ * r1 = kernel space destination address
+ * r2 = length
+ *
+ * Copies bytes from kernel space to kernel space, aborting on page fault
+ */
+ENTRY(kcopy)
+ cmp r2, #0x00
+#if /* XXX or <= 0 like below? */ 1
+ moveq r0, #0
+ moveq pc, lr
+#else
+ movle r0, #0x00
+ RETc(le) /* Bail early if length is <= 0 */
+#endif
+
+ push {r10-r11, lr}
+
+ /* Get curcpu from TPIDRPRW. */
+ mrc CP15_TPIDRPRW(r10)
+ ldr r10, [r10, #CI_CURPCB]
+
+ mov r3, #0x00
+ adr ip, .Lkcopy_fault
+ ldr r11, [r10, #PCB_ONFAULT]
+ str ip, [r10, #PCB_ONFAULT]
+ bl .Lkcopy_guts
+ str r11, [r10, #PCB_ONFAULT]
+ mov r0, #0x00
+ pop {r10-r11, pc}
+
+.Lkcopy_fault:
+ str r11, [r10, #PCB_ONFAULT]
+ cmp r3, #0x00
+ popgt {r4-r7} /* r3 > 0 Restore r4-r7 */
+ poplt {r4-r9} /* r3 < 0 Restore r4-r9 */
+ pop {r10-r11, pc}
+
+.Lkcopy_guts:
+ pld [r0]
+ /* Word-align the destination buffer */
+ ands ip, r1, #0x03 /* Already word aligned? */
+ beq .Lkcopy_wordaligned /* Yup */
+ rsb ip, ip, #0x04
+ cmp r2, ip /* Enough bytes left to align it? */
+ blt .Lkcopy_bad_endgame2 /* Nope. Just copy bytewise */
+ sub r2, r2, ip
+ rsbs ip, ip, #0x03
+ addne pc, pc, ip, lsl #3
+ nop
+ ldrb ip, [r0], #0x01
+ strb ip, [r1], #0x01
+ ldrb ip, [r0], #0x01
+ strb ip, [r1], #0x01
+ ldrb ip, [r0], #0x01
+ strb ip, [r1], #0x01
+ cmp r2, #0x00 /* All done? */
+ RETc(eq)
+
+ /* Destination buffer is now word aligned */
+.Lkcopy_wordaligned:
+ ands ip, r0, #0x03 /* Is src also word-aligned? */
+ bne .Lkcopy_bad_align /* Nope. Things just got bad */
+ cmp r2, #0x08 /* Less than 8 bytes remaining? */
+ blt .Lkcopy_w_less_than8
+
+ /* Quad-align the destination buffer */
+ tst r1, #0x07 /* Already quad aligned? */
+ ldrne ip, [r0], #0x04
+ push {r4-r9} /* Free up some registers */
+ mov r3, #-1 /* Signal restore r4-r9 */
+ subne r2, r2, #0x04
+ strne ip, [r1], #0x04
+
+ /* Destination buffer quad aligned, source is word aligned */
+ subs r2, r2, #0x80
+ blt .Lkcopy_w_lessthan128
+
+ /* Copy 128 bytes at a time */
+.Lkcopy_w_loop128:
+ ldr r4, [r0], #0x04 /* LD:00-03 */
+ ldr r5, [r0], #0x04 /* LD:04-07 */
+ pld [r0, #0x18] /* Prefetch 0x20 */
+ ldr r6, [r0], #0x04 /* LD:08-0b */
+ ldr r7, [r0], #0x04 /* LD:0c-0f */
+ ldr r8, [r0], #0x04 /* LD:10-13 */
+ ldr r9, [r0], #0x04 /* LD:14-17 */
+ strd r4, r5, [r1], #0x08 /* ST:00-07 */
+ ldr r4, [r0], #0x04 /* LD:18-1b */
+ ldr r5, [r0], #0x04 /* LD:1c-1f */
+ strd r6, r7, [r1], #0x08 /* ST:08-0f */
+ ldr r6, [r0], #0x04 /* LD:20-23 */
+ ldr r7, [r0], #0x04 /* LD:24-27 */
+ pld [r0, #0x18] /* Prefetch 0x40 */
+ strd r8, r9, [r1], #0x08 /* ST:10-17 */
+ ldr r8, [r0], #0x04 /* LD:28-2b */
+ ldr r9, [r0], #0x04 /* LD:2c-2f */
+ strd r4, r5, [r1], #0x08 /* ST:18-1f */
+ ldr r4, [r0], #0x04 /* LD:30-33 */
+ ldr r5, [r0], #0x04 /* LD:34-37 */
+ strd r6, r7, [r1], #0x08 /* ST:20-27 */
+ ldr r6, [r0], #0x04 /* LD:38-3b */
+ ldr r7, [r0], #0x04 /* LD:3c-3f */
+ strd r8, r9, [r1], #0x08 /* ST:28-2f */
+ ldr r8, [r0], #0x04 /* LD:40-43 */
+ ldr r9, [r0], #0x04 /* LD:44-47 */
+ pld [r0, #0x18] /* Prefetch 0x60 */
+ strd r4, r5, [r1], #0x08 /* ST:30-37 */
+ ldr r4, [r0], #0x04 /* LD:48-4b */
+ ldr r5, [r0], #0x04 /* LD:4c-4f */
+ strd r6, r7, [r1], #0x08 /* ST:38-3f */
+ ldr r6, [r0], #0x04 /* LD:50-53 */
+ ldr r7, [r0], #0x04 /* LD:54-57 */
+ strd r8, r9, [r1], #0x08 /* ST:40-47 */
+ ldr r8, [r0], #0x04 /* LD:58-5b */
+ ldr r9, [r0], #0x04 /* LD:5c-5f */
+ strd r4, r5, [r1], #0x08 /* ST:48-4f */
+ ldr r4, [r0], #0x04 /* LD:60-63 */
+ ldr r5, [r0], #0x04 /* LD:64-67 */
+ pld [r0, #0x18] /* Prefetch 0x80 */
+ strd r6, r7, [r1], #0x08 /* ST:50-57 */
+ ldr r6, [r0], #0x04 /* LD:68-6b */
+ ldr r7, [r0], #0x04 /* LD:6c-6f */
+ strd r8, r9, [r1], #0x08 /* ST:58-5f */
+ ldr r8, [r0], #0x04 /* LD:70-73 */
+ ldr r9, [r0], #0x04 /* LD:74-77 */
+ strd r4, r5, [r1], #0x08 /* ST:60-67 */
+ ldr r4, [r0], #0x04 /* LD:78-7b */
+ ldr r5, [r0], #0x04 /* LD:7c-7f */
+ strd r6, r7, [r1], #0x08 /* ST:68-6f */
+ strd r8, r9, [r1], #0x08 /* ST:70-77 */
+ subs r2, r2, #0x80
+ strd r4, r5, [r1], #0x08 /* ST:78-7f */
+ bge .Lkcopy_w_loop128
+
+.Lkcopy_w_lessthan128:
+ adds r2, r2, #0x80 /* Adjust for extra sub */
+ popeq {r4-r9}
+ moveq pc, lr /* Return now if done */
+ subs r2, r2, #0x20
+ blt .Lkcopy_w_lessthan32
+
+ /* Copy 32 bytes at a time */
+.Lkcopy_w_loop32:
+ ldr r4, [r0], #0x04
+ ldr r5, [r0], #0x04
+ pld [r0, #0x18]
+ ldr r6, [r0], #0x04
+ ldr r7, [r0], #0x04
+ ldr r8, [r0], #0x04
+ ldr r9, [r0], #0x04
+ strd r4, r5, [r1], #0x08
+ ldr r4, [r0], #0x04
+ ldr r5, [r0], #0x04
+ strd r6, r7, [r1], #0x08
+ strd r8, r9, [r1], #0x08
+ subs r2, r2, #0x20
+ strd r4, r5, [r1], #0x08
+ bge .Lkcopy_w_loop32
+
+.Lkcopy_w_lessthan32:
+ adds r2, r2, #0x20 /* Adjust for extra sub */
+ popeq {r4-r9}
+ moveq pc, lr /* Return now if done */
+
+ and r4, r2, #0x18
+ rsb r5, r4, #0x18
+ subs r2, r2, r4
+ add pc, pc, r5, lsl #1
+ nop
+
+ /* At least 24 bytes remaining */
+ ldr r4, [r0], #0x04
+ ldr r5, [r0], #0x04
+ nop
+ strd r4, r5, [r1], #0x08
+
+ /* At least 16 bytes remaining */
+ ldr r4, [r0], #0x04
+ ldr r5, [r0], #0x04
+ nop
+ strd r4, r5, [r1], #0x08
+
+ /* At least 8 bytes remaining */
+ ldr r4, [r0], #0x04
+ ldr r5, [r0], #0x04
+ nop
+ strd r4, r5, [r1], #0x08
+
+ /* Less than 8 bytes remaining */
+ pop {r4-r9}
+ moveq pc, lr /* Return now if done */
+ mov r3, #0x00
+
+.Lkcopy_w_less_than8:
+ subs r2, r2, #0x04
+ ldrge ip, [r0], #0x04
+ strge ip, [r1], #0x04
+ moveq pc, lr /* Return now if done */
+ addlt r2, r2, #0x04
+ ldrb ip, [r0], #0x01
+ cmp r2, #0x02
+ ldrbge r2, [r0], #0x01
+ strb ip, [r1], #0x01
+ ldrbgt ip, [r0]
+ strbge r2, [r1], #0x01
+ strbgt ip, [r1]
+ RET
+
+/*
+ * At this point, it has not been possible to word align both buffers.
+ * The destination buffer (r1) is word aligned, but the source buffer
+ * (r0) is not.
+ */
+.Lkcopy_bad_align:
+ push {r4-r7}
+ mov r3, #0x01
+ bic r0, r0, #0x03
+ cmp ip, #2
+ ldr ip, [r0], #0x04
+ bgt .Lkcopy_bad3
+ beq .Lkcopy_bad2
+ b .Lkcopy_bad1
+
+.Lkcopy_bad1_loop16:
+ mov r4, ip, lsr #8
+ ldr r5, [r0], #0x04
+ pld [r0, #0x018]
+ ldr r6, [r0], #0x04
+ ldr r7, [r0], #0x04
+ ldr ip, [r0], #0x04
+ orr r4, r4, r5, lsl #24
+ mov r5, r5, lsr #8
+ orr r5, r5, r6, lsl #24
+ mov r6, r6, lsr #8
+ orr r6, r6, r7, lsl #24
+ mov r7, r7, lsr #8
+ orr r7, r7, ip, lsl #24
+ str r4, [r1], #0x04
+ str r5, [r1], #0x04
+ str r6, [r1], #0x04
+ str r7, [r1], #0x04
+.Lkcopy_bad1:
+ subs r2, r2, #0x10
+ bge .Lkcopy_bad1_loop16
+
+ adds r2, r2, #0x10
+ popeq {r4-r7}
+ moveq pc, lr /* Return now if done */
+ subs r2, r2, #0x04
+ sublt r0, r0, #0x03
+ blt .Lkcopy_bad_endgame
+
+.Lkcopy_bad1_loop4:
+ mov r4, ip, lsr #8
+ ldr ip, [r0], #0x04
+ subs r2, r2, #0x04
+ orr r4, r4, ip, lsl #24
+ str r4, [r1], #0x04
+ bge .Lkcopy_bad1_loop4
+ sub r0, r0, #0x03
+ b .Lkcopy_bad_endgame
+
+.Lkcopy_bad2_loop16:
+ mov r4, ip, lsr #16
+ ldr r5, [r0], #0x04
+ pld [r0, #0x018]
+ ldr r6, [r0], #0x04
+ ldr r7, [r0], #0x04
+ ldr ip, [r0], #0x04
+ orr r4, r4, r5, lsl #16
+ mov r5, r5, lsr #16
+ orr r5, r5, r6, lsl #16
+ mov r6, r6, lsr #16
+ orr r6, r6, r7, lsl #16
+ mov r7, r7, lsr #16
+ orr r7, r7, ip, lsl #16
+ str r4, [r1], #0x04
+ str r5, [r1], #0x04
+ str r6, [r1], #0x04
+ str r7, [r1], #0x04
+.Lkcopy_bad2:
+ subs r2, r2, #0x10
+ bge .Lkcopy_bad2_loop16
+
+ adds r2, r2, #0x10
+ popeq {r4-r7}
+ moveq pc, lr /* Return now if done */
+ subs r2, r2, #0x04
+ sublt r0, r0, #0x02
+ blt .Lkcopy_bad_endgame
+
+.Lkcopy_bad2_loop4:
+ mov r4, ip, lsr #16
+ ldr ip, [r0], #0x04
+ subs r2, r2, #0x04
+ orr r4, r4, ip, lsl #16
+ str r4, [r1], #0x04
+ bge .Lkcopy_bad2_loop4
+ sub r0, r0, #0x02
+ b .Lkcopy_bad_endgame
+
+.Lkcopy_bad3_loop16:
+ mov r4, ip, lsr #24
+ ldr r5, [r0], #0x04
+ pld [r0, #0x018]
+ ldr r6, [r0], #0x04
+ ldr r7, [r0], #0x04
+ ldr ip, [r0], #0x04
+ orr r4, r4, r5, lsl #8
+ mov r5, r5, lsr #24
+ orr r5, r5, r6, lsl #8
+ mov r6, r6, lsr #24
+ orr r6, r6, r7, lsl #8
+ mov r7, r7, lsr #24
+ orr r7, r7, ip, lsl #8
+ str r4, [r1], #0x04
+ str r5, [r1], #0x04
+ str r6, [r1], #0x04
+ str r7, [r1], #0x04
+.Lkcopy_bad3:
+ subs r2, r2, #0x10
+ bge .Lkcopy_bad3_loop16
+
+ adds r2, r2, #0x10
+ popeq {r4-r7}
+ moveq pc, lr /* Return now if done */
+ subs r2, r2, #0x04
+ sublt r0, r0, #0x01
+ blt .Lkcopy_bad_endgame
+
+.Lkcopy_bad3_loop4:
+ mov r4, ip, lsl #24
+ ldr ip, [r0], #0x04
+ subs r2, r2, #0x04
+ orr r4, r4, ip, lsl #8
+ str r4, [r1], #0x04
+ bge .Lkcopy_bad3_loop4
+ sub r0, r0, #0x01
+
+.Lkcopy_bad_endgame:
+ pop {r4-r7}
+ mov r3, #0x00
+ adds r2, r2, #0x04
+ moveq pc, lr
+.Lkcopy_bad_endgame2:
+ rsbs r2, r2, #0x03
+ addne pc, pc, r2, lsl #3
+ nop
+ ldrb ip, [r0], #0x01
+ strb ip, [r1], #0x01
+ ldrb ip, [r0], #0x01
+ strb ip, [r1], #0x01
+ ldrb ip, [r0]
+ strb ip, [r1]
+ RET
+END(kcopy)
