From: Sebastian Andrzej Siewior <sebast...@breakpoint.cc> After seeing the numbers of of openssl's sha256 vs intree, vs crc64 vs none I decided to look at xxHash 16byte version which got stable in the 0.8 version (given I understood the signs right):
| Performance counter stats for './src/xz/.libs/xz -t xxh3.xz' (5 runs): | | 8.133.583.414 cycles # 4,293 GHz ( +- 0,01% ) (83,22%) | 14.366.854.241 instructions # 1,77 insn per cycle | | 1,8958 +- 0,0135 seconds time elapsed ( +- 0,71% ) So this is better than crc64 and close to none while doing something ;) Signed-off-by: Sebastian Andrzej Siewior <sebast...@breakpoint.cc> --- configure.ac | 6 +++++- src/liblzma/Makefile.am | 2 +- src/liblzma/api/lzma/check.h | 7 +++++++ src/liblzma/check/check.c | 27 +++++++++++++++++++++++++-- src/liblzma/check/check.h | 29 +++++++++++++++++++++++++++-- src/lzmainfo/Makefile.am | 2 +- src/xz/Makefile.am | 2 +- src/xz/args.c | 1 + src/xz/list.c | 2 +- src/xz/message.c | 2 +- src/xzdec/Makefile.am | 2 +- 11 files changed, 71 insertions(+), 11 deletions(-) diff --git a/configure.ac b/configure.ac index 5e0eaefc99c92..2e2bfe113da56 100644 --- a/configure.ac +++ b/configure.ac @@ -234,7 +234,7 @@ fi # Integrity checks # #################### -m4_define([SUPPORTED_CHECKS], [crc32,crc64,sha256]) +m4_define([SUPPORTED_CHECKS], [crc32,crc64,xxh3,sha256]) m4_foreach([NAME], [SUPPORTED_CHECKS], [enable_check_[]NAME=no @@ -798,6 +798,10 @@ if test "x$enable_openssl$openssl_found" = xyesno; then AC_MSG_ERROR([--enable-openssl was specified but openssl was not found.]) fi +if test "x$enable_check_xxh" = "xyes"; then + PKG_CHECK_MODULES([LIBXXHASH], [libxxhash >= 0.8.0]) +fi + # Check for SSE2 intrinsics. AC_CHECK_DECL([_mm_movemask_epi8], [AC_DEFINE([HAVE__MM_MOVEMASK_EPI8], [1], diff --git a/src/liblzma/Makefile.am b/src/liblzma/Makefile.am index 3afb08169840c..570a4db23c3f4 100644 --- a/src/liblzma/Makefile.am +++ b/src/liblzma/Makefile.am @@ -25,7 +25,7 @@ liblzma_la_CPPFLAGS = \ -I$(top_srcdir)/src/common \ -DTUKLIB_SYMBOL_PREFIX=lzma_ liblzma_la_LDFLAGS = -no-undefined -version-info 8:99:3 -liblzma_la_LDFLAGS += $(OPENSSL_CRYPTO_LIBS) +liblzma_la_LDFLAGS += $(OPENSSL_CRYPTO_LIBS) $(LIBXXHASH_LIBS) EXTRA_DIST += liblzma.map validate_map.sh if COND_SYMVERS diff --git a/src/liblzma/api/lzma/check.h b/src/liblzma/api/lzma/check.h index 6a243db0d7943..21aa6c8e3f7c9 100644 --- a/src/liblzma/api/lzma/check.h +++ b/src/liblzma/api/lzma/check.h @@ -46,6 +46,13 @@ typedef enum { * Size of the Check field: 8 bytes */ + LZMA_CHECK_XXH3 = 7, + /**< + * xxHash family, XXH3, 128bit + * + * Size of the Check field: 16 bytes + */ + LZMA_CHECK_SHA256 = 10 /**< * SHA-256 diff --git a/src/liblzma/check/check.c b/src/liblzma/check/check.c index 428ddaeb77981..2e168ae274c1f 100644 --- a/src/liblzma/check/check.c +++ b/src/liblzma/check/check.c @@ -39,7 +39,13 @@ lzma_check_is_supported(lzma_check type) false, // Reserved false, // Reserved - false, // Reserved + +#ifdef HAVE_CHECK_XXH3 + true, +#else + false, +#endif + false, // Reserved false, // Reserved @@ -48,7 +54,6 @@ lzma_check_is_supported(lzma_check type) #else false, #endif - false, // Reserved false, // Reserved false, // Reserved @@ -99,6 +104,12 @@ lzma_check_init(lzma_check_state *check, lzma_check type) break; #endif +#ifdef HAVE_CHECK_XXH3 + case LZMA_CHECK_XXH3: + lzma_xxh3_init(check); + break; +#endif + #ifdef HAVE_CHECK_SHA256 case LZMA_CHECK_SHA256: lzma_sha256_init(check); @@ -130,6 +141,12 @@ lzma_check_update(lzma_check_state *check, lzma_check type, break; #endif +#ifdef HAVE_CHECK_XXH3 + case LZMA_CHECK_XXH3: + lzma_xxh3_update(buf, size, check); + break; +#endif + #ifdef HAVE_CHECK_SHA256 case LZMA_CHECK_SHA256: lzma_sha256_update(buf, size, check); @@ -160,6 +177,12 @@ lzma_check_finish(lzma_check_state *check, lzma_check type) break; #endif +#ifdef HAVE_CHECK_XXH3 + case LZMA_CHECK_XXH3: + lzma_xxh3_finish(check); + break; +#endif + #ifdef HAVE_CHECK_SHA256 case LZMA_CHECK_SHA256: lzma_sha256_finish(check); diff --git a/src/liblzma/check/check.h b/src/liblzma/check/check.h index 0249025ec179a..910dc3d55fdca 100644 --- a/src/liblzma/check/check.h +++ b/src/liblzma/check/check.h @@ -39,6 +39,10 @@ # include <openssl/evp.h> #endif +#ifdef HAVE_CHECK_XXH3 +#include <xxhash.h> +#endif + #if defined(HAVE_INTERNAL_SHA256) /// State for the internal SHA-256 implementation typedef struct { @@ -99,6 +103,7 @@ typedef struct { uint32_t crc32; uint64_t crc64; lzma_sha256_state sha256; + XXH3_state_t *xxh3; } state; } lzma_check_state; @@ -191,11 +196,31 @@ lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check) LZMA_SHA256FUNC(Update)(&check->state.sha256, buf, size); } +#endif + +#ifdef HAVE_CHECK_XXH3 static inline void -lzma_sha256_finish(lzma_check_state *check) +lzma_xxh3_init(lzma_check_state *check) { - LZMA_SHA256FUNC(Final)(check->buffer.u8, &check->state.sha256); + check->state.xxh3 = XXH3_createState(); + XXH3_128bits_reset(check->state.xxh3); +} + +static inline void +lzma_xxh3_update(const uint8_t *buf, size_t size, lzma_check_state *check) +{ + XXH3_128bits_update(check->state.xxh3, buf, size); +} + +static inline void +lzma_xxh3_finish(lzma_check_state *check) +{ + XXH128_hash_t h; + + h = XXH3_128bits_digest(check->state.xxh3); + memcpy(check->buffer.u8, &h, sizeof(h)); + XXH3_freeState(check->state.xxh3); } #endif diff --git a/src/lzmainfo/Makefile.am b/src/lzmainfo/Makefile.am index 51424effb750a..5f12138a1914d 100644 --- a/src/lzmainfo/Makefile.am +++ b/src/lzmainfo/Makefile.am @@ -28,7 +28,7 @@ if COND_GNULIB lzmainfo_LDADD += $(top_builddir)/lib/libgnu.a endif -lzmainfo_LDADD += $(LTLIBINTL) $(OPENSSL_CRYPTO_LIBS) +lzmainfo_LDADD += $(LTLIBINTL) $(OPENSSL_CRYPTO_LIBS) $(LIBXXHASH_LIBS) dist_man_MANS = lzmainfo.1 diff --git a/src/xz/Makefile.am b/src/xz/Makefile.am index 8d6604c8ffb22..55e412ecc5f16 100644 --- a/src/xz/Makefile.am +++ b/src/xz/Makefile.am @@ -60,7 +60,7 @@ xz_LDADD += $(top_builddir)/lib/libgnu.a endif # libgnu.a may need these libs, so this must be after libgnu.a. -xz_LDADD += $(LTLIBINTL) $(OPENSSL_CRYPTO_LIBS) +xz_LDADD += $(LTLIBINTL) $(OPENSSL_CRYPTO_LIBS) $(LIBXXHASH_LIBS) # Windows resource compiler support diff --git a/src/xz/args.c b/src/xz/args.c index 9238fb32ec00d..404ab5889342f 100644 --- a/src/xz/args.c +++ b/src/xz/args.c @@ -421,6 +421,7 @@ parse_real(args_info *args, int argc, char **argv) { "crc32", LZMA_CHECK_CRC32 }, { "crc64", LZMA_CHECK_CRC64 }, { "sha256", LZMA_CHECK_SHA256 }, + { "xxh3", LZMA_CHECK_XXH3 }, }; size_t i = 0; diff --git a/src/xz/list.c b/src/xz/list.c index 06c9c1ee20e74..778f5283682c5 100644 --- a/src/xz/list.c +++ b/src/xz/list.c @@ -174,7 +174,7 @@ static const char check_names[LZMA_CHECK_ID_MAX + 1][12] = { "CRC64", N_("Unknown-5"), N_("Unknown-6"), - N_("Unknown-7"), + N_("XXH3"), N_("Unknown-8"), N_("Unknown-9"), "SHA-256", diff --git a/src/xz/message.c b/src/xz/message.c index 00eb65b62c5cc..93dcde5ea8831 100644 --- a/src/xz/message.c +++ b/src/xz/message.c @@ -1145,7 +1145,7 @@ message_help(bool long_help) " -F, --format=FMT file format to encode or decode; possible values are\n" " `auto' (default), `xz', `lzma', and `raw'\n" " -C, --check=CHECK integrity check type: `none' (use with caution),\n" -" `crc32', `crc64' (default), or `sha256'")); +" `crc32', `crc64' (default), `xxh3' or `sha256'")); puts(_( " --ignore-check don't verify the integrity check when decompressing")); } diff --git a/src/xzdec/Makefile.am b/src/xzdec/Makefile.am index 59ad965eb1678..b2cf799402dea 100644 --- a/src/xzdec/Makefile.am +++ b/src/xzdec/Makefile.am @@ -32,7 +32,7 @@ if COND_GNULIB xzdec_LDADD += $(top_builddir)/lib/libgnu.a endif -xzdec_LDADD += $(LTLIBINTL) $(OPENSSL_CRYPTO_LIBS) +xzdec_LDADD += $(LTLIBINTL) $(OPENSSL_CRYPTO_LIBS) $(LIBXXHASH_LIBS) lzmadec_SOURCES = \ -- 2.30.0