Author: mjg Date: Thu Sep 27 17:08:29 2018 New Revision: 338973 URL: https://svnweb.freebsd.org/changeset/base/338973
Log: amd64: reimplement libc memcmp and bcmp with kernel memcmp Both are significantly slower than hand-coded loops. See r338963 for kernel commit. bcmp differs from memcmp by always returning 1 when a difference is found, as opposed to going for a value bigger or lower than 0 depending on what it is. This means it can do less work. For now the code is duplicated and modified. This will get deduplicated after another round of optimization when memcmp will get a longer-term form. Both tested with the glibc suite. While the suite does not have a test for bcmp, I created a wrapper routine which verified that values match (0 vs 0, 1 vs non-zero). Reviewed by: kib Approved by: re (gjb) Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D17336 Modified: head/lib/libc/amd64/string/bcmp.S head/lib/libc/amd64/string/memcmp.S Modified: head/lib/libc/amd64/string/bcmp.S ============================================================================== --- head/lib/libc/amd64/string/bcmp.S Thu Sep 27 16:43:41 2018 (r338972) +++ head/lib/libc/amd64/string/bcmp.S Thu Sep 27 17:08:29 2018 (r338973) @@ -1,27 +1,121 @@ +/*- + * Copyright (c) 2018 The FreeBSD Foundation + * + * This software was developed by Mateusz Guzik <m...@freebsd.org> + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + #include <machine/asm.h> __FBSDID("$FreeBSD$"); -#if 0 - RCSID("$NetBSD: bcmp.S,v 1.1 2001/06/19 00:25:04 fvdl Exp $") -#endif - ENTRY(bcmp) - cld /* set compare direction forward */ + cmpq $16,%rdx + jae 5f +1: + testq %rdx,%rdx + je 3f + xorl %ecx,%ecx +2: + movzbl (%rdi,%rcx,1),%eax + movzbl (%rsi,%rcx,1),%r8d + cmpb %r8b,%al + jne 4f + addq $1,%rcx + cmpq %rcx,%rdx + jz 3f + movzbl (%rdi,%rcx,1),%eax + movzbl (%rsi,%rcx,1),%r8d + cmpb %r8b,%al + jne 4f + addq $1,%rcx + cmpq %rcx,%rdx + jz 3f + movzbl (%rdi,%rcx,1),%eax + movzbl (%rsi,%rcx,1),%r8d + cmpb %r8b,%al + jne 4f + addq $1,%rcx + cmpq %rcx,%rdx + jz 3f + movzbl (%rdi,%rcx,1),%eax + movzbl (%rsi,%rcx,1),%r8d + cmpb %r8b,%al + jne 4f + addq $1,%rcx + cmpq %rcx,%rdx + jne 2b +3: + xorl %eax,%eax + ret +4: + movl $1,%eax + ret +5: + cmpq $32,%rdx + jae 7f +6: + /* + * 8 bytes + */ + movq (%rdi),%r8 + movq (%rsi),%r9 + cmpq %r8,%r9 + jne 4b + leaq 8(%rdi),%rdi + leaq 8(%rsi),%rsi + subq $8,%rdx + cmpq $8,%rdx + jae 6b + jl 1b + jmp 3b +7: + /* + * 32 bytes + */ + movq (%rsi),%r8 + movq 8(%rsi),%r9 + subq (%rdi),%r8 + subq 8(%rdi),%r9 + or %r8,%r9 + jnz 4b - movq %rdx,%rcx /* compare by words */ - shrq $3,%rcx - repe - cmpsq - jne L1 + movq 16(%rsi),%r8 + movq 24(%rsi),%r9 + subq 16(%rdi),%r8 + subq 24(%rdi),%r9 + or %r8,%r9 + jnz 4b - movq %rdx,%rcx /* compare remainder by bytes */ - andq $7,%rcx - repe - cmpsb -L1: - setne %al - movsbl %al,%eax - ret + leaq 32(%rdi),%rdi + leaq 32(%rsi),%rsi + subq $32,%rdx + cmpq $32,%rdx + jae 7b + jnz 1b + jmp 3b END(bcmp) .section .note.GNU-stack,"",%progbits Modified: head/lib/libc/amd64/string/memcmp.S ============================================================================== --- head/lib/libc/amd64/string/memcmp.S Thu Sep 27 16:43:41 2018 (r338972) +++ head/lib/libc/amd64/string/memcmp.S Thu Sep 27 17:08:29 2018 (r338973) @@ -1,44 +1,121 @@ -/* - * Written by J.T. Conklin <j...@netbsd.org>. - * Public domain. - * Adapted for NetBSD/x86_64 by Frank van der Linden <f...@wasabisystems.com> +/*- + * Copyright (c) 2018 The FreeBSD Foundation + * + * This software was developed by Mateusz Guzik <m...@freebsd.org> + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ */ #include <machine/asm.h> __FBSDID("$FreeBSD$"); -#if 0 - RCSID("$NetBSD: memcmp.S,v 1.2 2003/07/26 19:24:39 salo Exp $") -#endif - ENTRY(memcmp) - cld /* set compare direction forward */ - movq %rdx,%rcx /* compare by longs */ - shrq $3,%rcx - repe - cmpsq - jne L5 /* do we match so far? */ - - movq %rdx,%rcx /* compare remainder by bytes */ - andq $7,%rcx - repe - cmpsb - jne L6 /* do we match? */ - - xorl %eax,%eax /* we match, return zero */ + cmpq $16,%rdx + jae 5f +1: + testq %rdx,%rdx + je 3f + xorl %ecx,%ecx +2: + movzbl (%rdi,%rcx,1),%eax + movzbl (%rsi,%rcx,1),%r8d + cmpb %r8b,%al + jne 4f + addq $1,%rcx + cmpq %rcx,%rdx + jz 3f + movzbl (%rdi,%rcx,1),%eax + movzbl (%rsi,%rcx,1),%r8d + cmpb %r8b,%al + jne 4f + addq $1,%rcx + cmpq %rcx,%rdx + jz 3f + movzbl (%rdi,%rcx,1),%eax + movzbl (%rsi,%rcx,1),%r8d + cmpb %r8b,%al + jne 4f + addq $1,%rcx + cmpq %rcx,%rdx + jz 3f + movzbl (%rdi,%rcx,1),%eax + movzbl (%rsi,%rcx,1),%r8d + cmpb %r8b,%al + jne 4f + addq $1,%rcx + cmpq %rcx,%rdx + jne 2b +3: + xorl %eax,%eax ret - -L5: movl $8,%ecx /* We know that one of the next */ - subq %rcx,%rdi /* eight pairs of bytes do not */ - subq %rcx,%rsi /* match. */ - repe - cmpsb -L6: xorl %eax,%eax /* Perform unsigned comparison */ - movb -1(%rdi),%al - xorl %edx,%edx - movb -1(%rsi),%dl - subl %edx,%eax +4: + subl %r8d,%eax ret +5: + cmpq $32,%rdx + jae 7f +6: + /* + * 8 bytes + */ + movq (%rdi),%r8 + movq (%rsi),%r9 + cmpq %r8,%r9 + jne 1b + leaq 8(%rdi),%rdi + leaq 8(%rsi),%rsi + subq $8,%rdx + cmpq $8,%rdx + jae 6b + jl 1b + jmp 3b +7: + /* + * 32 bytes + */ + movq (%rsi),%r8 + movq 8(%rsi),%r9 + subq (%rdi),%r8 + subq 8(%rdi),%r9 + or %r8,%r9 + jnz 1b + + movq 16(%rsi),%r8 + movq 24(%rsi),%r9 + subq 16(%rdi),%r8 + subq 24(%rdi),%r9 + or %r8,%r9 + jnz 1b + + leaq 32(%rdi),%rdi + leaq 32(%rsi),%rsi + subq $32,%rdx + cmpq $32,%rdx + jae 7b + jnz 1b + jmp 3b END(memcmp) .section .note.GNU-stack,"",%progbits _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
