Author: tsoome Date: Sat Nov 2 12:28:04 2019 New Revision: 354253 URL: https://svnweb.freebsd.org/changeset/base/354253
Log: Remove duplicate lz4 implementations Port illumos change: https://www.illumos.org/issues/11667 Move lz4.c out of zfs tree to opensolaris/common/lz4, adjust it to be usable from kernel/stand/userland builds, so we can use just one single source. Add lz4.h to declare lz4_compress() and lz4_decompress(). MFC after: 1 month Differential Revision: https://reviews.freebsd.org/D22037 Added: head/sys/cddl/contrib/opensolaris/common/lz4/ head/sys/cddl/contrib/opensolaris/common/lz4/lz4.c - copied, changed from r354252, head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/lz4.c head/sys/cddl/contrib/opensolaris/common/lz4/lz4.h (contents, props changed) Deleted: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/lz4.c Modified: head/cddl/lib/libzpool/Makefile head/stand/efi/boot1/Makefile head/stand/i386/gptzfsboot/Makefile head/stand/i386/zfsboot/Makefile head/stand/libsa/Makefile head/stand/libsa/zfs/Makefile.inc head/sys/cddl/boot/zfs/zfssubr.c head/sys/cddl/contrib/opensolaris/uts/common/Makefile.files head/sys/modules/zfs/Makefile Modified: head/cddl/lib/libzpool/Makefile ============================================================================== --- head/cddl/lib/libzpool/Makefile Sat Nov 2 10:53:23 2019 (r354252) +++ head/cddl/lib/libzpool/Makefile Sat Nov 2 12:28:04 2019 (r354253) @@ -8,6 +8,8 @@ .PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/lua # ZFS_SHARED_SRCS .PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/common/zfs +# LZ4_COMMON_SRCS +.PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/common/lz4 # KERNEL_SRCS .PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libzpool/common # LIST_SRCS @@ -32,6 +34,7 @@ LIB= zpool ZFS_COMMON_SRCS= ${ZFS_COMMON_OBJS:C/.o$/.c/} trim_map.c ZFS_SHARED_SRCS= ${ZFS_SHARED_OBJS:C/.o$/.c/} +LZ4_COMMON_SRCS= lz4.c LUA_SRCS= ${LUA_OBJS:C/.o$/.c/} KERNEL_SRCS= kernel.c taskq.c util.c LIST_SRCS= list.c @@ -39,7 +42,7 @@ UNICODE_SRCS= u8_textprep.c LIBCMDUTILS_SRCS=nicenum.c SRCS= ${ZFS_COMMON_SRCS} ${ZFS_SHARED_SRCS} ${LUA_SRCS} \ - ${KERNEL_SRCS} ${LIST_SRCS} ${ATOMIC_SRCS} \ + ${LZ4_COMMON_SRCS} ${KERNEL_SRCS} ${LIST_SRCS} ${ATOMIC_SRCS} \ ${UNICODE_SRCS} ${LIBCMDUTILS_SRCS} WARNS?= 0 @@ -50,6 +53,7 @@ CFLAGS+= -I${SRCTOP}/cddl/contrib/opensolaris/lib/libz CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common/fs/zfs CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/lua CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/common/zfs +CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/common/lz4 CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common CFLAGS+= -I${SRCTOP}/cddl/contrib/opensolaris/head CFLAGS+= -I${SRCTOP}/cddl/contrib/opensolaris/lib/libnvpair Modified: head/stand/efi/boot1/Makefile ============================================================================== --- head/stand/efi/boot1/Makefile Sat Nov 2 10:53:23 2019 (r354252) +++ head/stand/efi/boot1/Makefile Sat Nov 2 12:28:04 2019 (r354253) @@ -37,6 +37,7 @@ CFLAGS.zfs_module.c+= -I${ZFSSRC} CFLAGS.zfs_module.c+= -I${SYSDIR}/cddl/boot/zfs CFLAGS.zfs_module.c+= -I${SYSDIR}/crypto/skein CFLAGS.zfs_module.c+= -I${SYSDIR}/cddl/contrib/opensolaris/uts/common +CFLAGS.zfs_module.c+= -I${SYSDIR}/cddl/contrib/opensolaris/common/lz4 CFLAGS+= -DEFI_ZFS_BOOT .endif Modified: head/stand/i386/gptzfsboot/Makefile ============================================================================== --- head/stand/i386/gptzfsboot/Makefile Sat Nov 2 10:53:23 2019 (r354252) +++ head/stand/i386/gptzfsboot/Makefile Sat Nov 2 12:28:04 2019 (r354253) @@ -29,6 +29,7 @@ CFLAGS+=-DBOOTPROG=\"gptzfsboot\" \ -I${SYSDIR}/crypto/skein \ -I${SYSDIR}/cddl/boot/zfs \ -I${SYSDIR}/cddl/contrib/opensolaris/uts/common \ + -I${SYSDIR}/cddl/contrib/opensolaris/common/lz4 \ -I${BOOTSRC}/i386/btx/lib \ -I${BOOTSRC}/i386/boot2 \ -Wall -Waggregate-return -Wbad-function-cast \ Modified: head/stand/i386/zfsboot/Makefile ============================================================================== --- head/stand/i386/zfsboot/Makefile Sat Nov 2 10:53:23 2019 (r354252) +++ head/stand/i386/zfsboot/Makefile Sat Nov 2 12:28:04 2019 (r354253) @@ -28,6 +28,7 @@ CFLAGS+=-DBOOTPROG=\"zfsboot\" \ -I${SYSDIR}/crypto/skein \ -I${SYSDIR}/cddl/boot/zfs \ -I${SYSDIR}/cddl/contrib/opensolaris/uts/common \ + -I${SYSDIR}/cddl/contrib/opensolaris/common/lz4 \ -I${BOOTSRC}/i386/boot2 \ -Wall -Waggregate-return -Wbad-function-cast -Wno-cast-align \ -Wmissing-declarations -Wmissing-prototypes -Wnested-externs \ Modified: head/stand/libsa/Makefile ============================================================================== --- head/stand/libsa/Makefile Sat Nov 2 10:53:23 2019 (r354252) +++ head/stand/libsa/Makefile Sat Nov 2 12:28:04 2019 (r354253) @@ -102,6 +102,11 @@ CFLAGS+=-DHAVE_MEMCPY -I${SRCTOP}/sys/contrib/zlib SRCS+= adler32.c crc32.c SRCS+= infback.c inffast.c inflate.c inftrees.c zutil.c +# lz4 decompression functionality +.PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/common/lz4 +SRCS+= lz4.c +CFLAGS.lz4.c+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/common/lz4 + # Create a subset of includes that are safe, as well as adjusting those that aren't # The lists may drive people nuts, but they are explicitly opt-in FAKE_DIRS=xlocale arpa Modified: head/stand/libsa/zfs/Makefile.inc ============================================================================== --- head/stand/libsa/zfs/Makefile.inc Sat Nov 2 10:53:23 2019 (r354252) +++ head/stand/libsa/zfs/Makefile.inc Sat Nov 2 12:28:04 2019 (r354253) @@ -12,4 +12,6 @@ CFLAGS+= -I${SYSDIR}/cddl/boot/zfs CFLAGS+= -I${SYSDIR}/cddl/contrib/opensolaris/uts/common CFLAGS+= -I${SYSDIR}/crypto/skein +CFLAGS.zfs.c+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/common/lz4 + CFLAGS+= -Wformat -Wall Modified: head/sys/cddl/boot/zfs/zfssubr.c ============================================================================== --- head/sys/cddl/boot/zfs/zfssubr.c Sat Nov 2 10:53:23 2019 (r354252) +++ head/sys/cddl/boot/zfs/zfssubr.c Sat Nov 2 12:28:04 2019 (r354253) @@ -26,6 +26,8 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); +#include <lz4.h> + static uint64_t zfs_crc64_table[256]; #define ECKSUM 666 @@ -161,7 +163,6 @@ typedef struct zio_compress_info { #include "lzjb.c" #include "zle.c" -#include "lz4.c" /* * Compression vectors. Copied and modified: head/sys/cddl/contrib/opensolaris/common/lz4/lz4.c (from r354252, head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/lz4.c) ============================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/lz4.c Sat Nov 2 10:53:23 2019 (r354252, copy source) +++ head/sys/cddl/contrib/opensolaris/common/lz4/lz4.c Sat Nov 2 12:28:04 2019 (r354253) @@ -35,11 +35,29 @@ * Copyright (c) 2016 by Delphix. All rights reserved. */ +#if defined(_KERNEL) #include <sys/zfs_context.h> +#elif defined(_STANDALONE) +#include <sys/cdefs.h> +#include <stand.h> +#include <sys/types.h> +#include <sys/endian.h> +#include <assert.h> +#define ASSERT assert +#else +#include <string.h> +#include <stdlib.h> +#include <sys/types.h> +#include <netinet/in.h> +#include <assert.h> + +#define ASSERT assert +#endif +#include <lz4.h> + static int real_LZ4_compress(const char *source, char *dest, int isize, int osize); -static int LZ4_compressBound(int isize); static int LZ4_uncompress_unknownOutputSize(const char *source, char *dest, int isize, int maxOutputSize); static int LZ4_compressCtx(void *ctx, const char *source, char *dest, @@ -47,11 +65,13 @@ static int LZ4_compressCtx(void *ctx, const char *sour static int LZ4_compress64kCtx(void *ctx, const char *source, char *dest, int isize, int osize); +#if defined(_KERNEL) static kmem_cache_t *lz4_ctx_cache; +#endif -/*ARGSUSED*/ size_t -lz4_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) +lz4_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, + int n __unused) { uint32_t bufsiz; char *dest = d_start; @@ -71,17 +91,25 @@ lz4_compress(void *s_start, void *d_start, size_t s_le * added to the compressed buffer and which, if unhandled, would * confuse the hell out of our decompression function. */ - *(uint32_t *)dest = BE_32(bufsiz); +#if defined(_KERNEL) + *(uint32_t *)(void *)dest = BE_32(bufsiz); +#else + *(uint32_t *)(void *)dest = htonl(bufsiz); +#endif return (bufsiz + sizeof (bufsiz)); } -/*ARGSUSED*/ int -lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) +lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, + int n __unused) { const char *src = s_start; - uint32_t bufsiz = BE_IN32(src); +#if defined(_KERNEL) + uint32_t bufsiz = BE_IN32(s_start); +#else + uint32_t bufsiz = htonl(*(uint32_t *)s_start); +#endif /* invalid compressed buffer size encoded at start */ if (bufsiz + sizeof (bufsiz) > s_len) @@ -105,20 +133,10 @@ lz4_decompress(void *s_start, void *d_start, size_t s_ * or 0 if the compression fails (if LZ4_COMPRESSMIN is set). * note : destination buffer must be already allocated. * destination buffer must be sized to handle worst cases - * situations (input data not compressible) worst case size - * evaluation is provided by function LZ4_compressBound(). + * situations (input data not compressible). * * Advanced Functions * - * LZ4_compressBound() : - * Provides the maximum size that LZ4 may output in a "worst case" - * scenario (input data not compressible) primarily useful for memory - * allocation of output buffer. - * - * isize : is the input size. Max supported value is ~1.9GB - * return : maximum output size in a "worst case" scenario - * note : this function is limited by "int" range (2^31-1) - * * LZ4_uncompress_unknownOutputSize() : * isize : is the input size, therefore the compressed size * maxOutputSize : is the size of the destination buffer (which must be @@ -292,9 +310,9 @@ typedef struct _U64_S { #pragma pack() #endif -#define A64(x) (((U64_S *)(x))->v) -#define A32(x) (((U32_S *)(x))->v) -#define A16(x) (((U16_S *)(x))->v) +#define A64(x) (((U64_S *)(__DECONST(void *, x)))->v) +#define A32(x) (((U32_S *)(__DECONST(void *, x)))->v) +#define A16(x) (((U16_S *)(__DECONST(void *, x)))->v) /* * Constants @@ -456,14 +474,6 @@ LZ4_NbCommonBytes(register U32 val) #endif -/* Public functions */ - -static int -LZ4_compressBound(int isize) -{ - return (isize + (isize / 255) + 16); -} - /* Compression functions */ /*ARGSUSED*/ @@ -478,7 +488,7 @@ LZ4_compressCtx(void *ctx, const char *source, char *d HTYPE HashTable[HASHTABLESIZE] = { 0 }; #endif - const BYTE *ip = (BYTE *) source; + const BYTE *ip = (const BYTE *) source; INITBASE(base); const BYTE *anchor = ip; const BYTE *const iend = ip + isize; @@ -527,7 +537,7 @@ LZ4_compressCtx(void *ctx, const char *source, char *d } while ((ref < ip - MAX_DISTANCE) || (A32(ref) != A32(ip))); /* Catch up */ - while ((ip > anchor) && (ref > (BYTE *) source) && + while ((ip > anchor) && (ref > (const BYTE *) source) && unlikely(ip[-1] == ref[-1])) { ip--; ref--; @@ -673,7 +683,7 @@ LZ4_compress64kCtx(void *ctx, const char *source, char U16 HashTable[HASH64KTABLESIZE] = { 0 }; #endif - const BYTE *ip = (BYTE *) source; + const BYTE *ip = (const BYTE *) source; const BYTE *anchor = ip; const BYTE *const base = ip; const BYTE *const iend = ip + isize; @@ -720,7 +730,7 @@ LZ4_compress64kCtx(void *ctx, const char *source, char } while (A32(ref) != A32(ip)); /* Catch up */ - while ((ip > anchor) && (ref > (BYTE *) source) && + while ((ip > anchor) && (ref > (const BYTE *) source) && (ip[-1] == ref[-1])) { ip--; ref--; @@ -847,7 +857,11 @@ static int real_LZ4_compress(const char *source, char *dest, int isize, int osize) { #if HEAPMODE +#if defined(_KERNEL) void *ctx = kmem_cache_alloc(lz4_ctx_cache, KM_NOSLEEP); +#else + void *ctx = malloc(sizeof(struct refTables)); +#endif int result; /* @@ -863,7 +877,11 @@ real_LZ4_compress(const char *source, char *dest, int else result = LZ4_compressCtx(ctx, source, dest, isize, osize); +#if defined(_KERNEL) kmem_cache_free(lz4_ctx_cache, ctx); +#else + free(ctx); +#endif return (result); #else if (isize < (int)LZ4_64KLIMIT) @@ -876,7 +894,7 @@ real_LZ4_compress(const char *source, char *dest, int /* * Note: The decoding function LZ4_uncompress_unknownOutputSize() is safe - * against "buffer overflow" attack type. They will never write nor + * against "buffer overflow" attack type. It will never write nor * read outside of the provided output buffers. * LZ4_uncompress_unknownOutputSize() also insures that it will never * read outside of the input buffer. A corrupted input will produce @@ -1010,9 +1028,10 @@ LZ4_uncompress_unknownOutputSize(const char *source, c /* write overflow error detected */ _output_error: - return (int)(-(((char *)ip) - source)); + return (int)(-(((const char *)ip) - source)); } +#if defined(_KERNEL) extern void lz4_init(void) { @@ -1031,3 +1050,4 @@ lz4_fini(void) kmem_cache_destroy(lz4_ctx_cache); #endif } +#endif /* _KERNEL */ Added: head/sys/cddl/contrib/opensolaris/common/lz4/lz4.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/cddl/contrib/opensolaris/common/lz4/lz4.h Sat Nov 2 12:28:04 2019 (r354253) @@ -0,0 +1,50 @@ +/* + * LZ4 - Fast LZ compression algorithm + * Header File + * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at : + * - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + * - LZ4 source repository : http://code.google.com/p/lz4/ + */ + +#ifndef _LZ4_H +#define _LZ4_H + +#include <sys/types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +extern size_t lz4_compress(void *, void *, size_t, size_t, int); +extern int lz4_decompress(void *, void *, size_t, size_t, int); + +#ifdef __cplusplus +} +#endif + +#endif /* _LZ4_H */ Modified: head/sys/cddl/contrib/opensolaris/uts/common/Makefile.files ============================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/Makefile.files Sat Nov 2 10:53:23 2019 (r354252) +++ head/sys/cddl/contrib/opensolaris/uts/common/Makefile.files Sat Nov 2 12:28:04 2019 (r354253) @@ -98,7 +98,6 @@ ZFS_COMMON_OBJS += \ dsl_scan.o \ zfeature.o \ gzip.o \ - lz4.o \ lzjb.o \ metaslab.o \ multilist.o \ Modified: head/sys/modules/zfs/Makefile ============================================================================== --- head/sys/modules/zfs/Makefile Sat Nov 2 10:53:23 2019 (r354252) +++ head/sys/modules/zfs/Makefile Sat Nov 2 12:28:04 2019 (r354253) @@ -18,6 +18,8 @@ SRCS+= opensolaris_nvpair_alloc_fixed.c SRCS+= opensolaris_fnvpair.c .PATH: ${SYSDIR}/cddl/contrib/opensolaris/common/unicode SRCS+= u8_textprep.c +.PATH: ${SUNW}/common/lz4 +SRCS+= lz4.c .PATH: ${SYSDIR}/cddl/compat/opensolaris/kern SRCS+= opensolaris_acl.c @@ -84,6 +86,7 @@ CFLAGS+=-I${SUNW}/uts/common/zmod CFLAGS+=-I${SUNW}/uts/common CFLAGS+=-I${SYSDIR} CFLAGS+=-I${SUNW}/common/zfs +CFLAGS+=-I${SUNW}/common/lz4 CFLAGS+=-I${SUNW}/common CFLAGS+=-DBUILDING_ZFS CFLAGS.gcc+=-fms-extensions _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
