This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch zstd-basic in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 08b4fc0645f42c11a63932a6df24683333301b17 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Tue Jun 9 01:35:18 2026 -0400 Basic zstd compression This is using the built-in zstd API from Erlang/OTP 28+. For 26 and 27 it falls back to `deflate`. Otherwise it's not that different than deflate and snappy which we already support. An interesting feature of zstd is to use dictionaries. This PR doesn't do it, but it sets up a shim `couch_zstd` module where that could plug-in eventually. The idea is to train a dictionary it based on document bodies for each database before compaction and then saving the dictionary. zstd as is in Erlang/OTP doesn't expose a trainign API but we can still use a concatenative/raw dictionary: ```erlang > Dict = ~"{\"type\":\"foo\",\"bar\":1,\"baz\":\"a\"}". <<"{\"type\":\"foo\",\"bar\":1,\"baz\":\"a\"}">> > {ok, C} = zstd:dict(compress, Dict). {ok,{cdict,#Ref<0.4075923125.4131520536.31325>}} > Doc = ~"{\"type\":\"foo\", \"bar\":2, \"baz\":\"b\"}". <<"{\"type\":\"foo\", \"bar\":2, \"baz\":\"b\"}">> > byte_size(iolist_to_binary(zstd:compress(Doc, #{dictionary=>C}))). 29 > byte_size(iolist_to_binary(zstd:compress(Doc))). 43 ``` --- rel/overlay/etc/default.ini | 2 + src/couch/src/couch_compress.erl | 43 ++++++++++++++--- src/couch/src/couch_zstd.erl | 56 ++++++++++++++++++++++ src/couch/test/eunit/couch_compress_tests.erl | 39 ++++++++++++++- .../test/eunit/couchdb_file_compression_tests.erl | 6 +++ 5 files changed, 139 insertions(+), 7 deletions(-) diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index 0662e6afd..aac8ff584 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -31,6 +31,8 @@ view_index_dir = {{view_index_dir}} ; snappy - use google snappy, a very fast compressor/decompressor ; deflate_N - use zlib's deflate, N is the compression level which ranges from 1 (fastest, ; lowest compression ratio) to 9 (slowest, highest compression ratio) +; zstd - Zstandard (needs Erlang 28+ falls back deflate_3). Defaults to level 3 +; zstd_N - Zstandard level -22..22 ;file_compression = snappy ; Higher values may give better read performance due to less read operations diff --git a/src/couch/src/couch_compress.erl b/src/couch/src/couch_compress.erl index 59d692058..336f26370 100644 --- a/src/couch/src/couch_compress.erl +++ b/src/couch/src/couch_compress.erl @@ -24,20 +24,37 @@ % http://www.erlang.org/doc/apps/erts/erl_ext_dist.html -define(TERM_PREFIX, 131). -define(COMPRESSED_TERM_PREFIX, 131, 80). +% Zstandard frame magic number = 0xFD2FB528 in little endian(!) +-define(ZSTD_MAGIC, 16#28, 16#B5, 16#2F, 16#FD). +-define(ZSTD_DEFAULT_LEVEL, 3). get_compression_method() -> case config:get("couchdb", "file_compression") of undefined -> ?DEFAULT_COMPRESSION; Method1 -> - case string:tokens(Method1, "_") of - [Method] -> - list_to_existing_atom(Method); - [Method, Level] -> - {list_to_existing_atom(Method), list_to_integer(Level)} - end + Method = + case string:tokens(Method1, "_") of + [M] -> + list_to_existing_atom(M); + [M, Level] -> + {list_to_existing_atom(M), list_to_integer(Level)} + end, + maybe_zstd(Method) end. +maybe_zstd(zstd) -> + maybe_zstd({zstd, ?ZSTD_DEFAULT_LEVEL}); +maybe_zstd({zstd, _} = Method) -> + % zstd needs OTP 28+. On older releases fall back to deflate at a + % comparable level (zstd's default is also 3) so writes still compress. + case couch_zstd:available() of + true -> Method; + false -> {deflate, 3} + end; +maybe_zstd(Method) -> + Method. + compress(<<?SNAPPY_PREFIX, _/binary>> = Bin, snappy) -> Bin; compress(<<?SNAPPY_PREFIX, _/binary>> = Bin, Method) -> @@ -46,10 +63,16 @@ compress(<<?COMPRESSED_TERM_PREFIX, _/binary>> = Bin, {deflate, _Level}) -> Bin; compress(<<?TERM_PREFIX, _/binary>> = Bin, Method) -> compress(decompress(Bin), Method); +compress(<<?ZSTD_MAGIC, _/binary>> = Bin, {zstd, _Level}) -> + Bin; +compress(<<?ZSTD_MAGIC, _/binary>> = Bin, Method) -> + compress(decompress(Bin), Method); compress(Term, none) -> ?term_to_bin(Term); compress(Term, {deflate, Level}) -> term_to_binary(Term, [{minor_version, 1}, {compressed, Level}]); +compress(Term, {zstd, Level}) -> + couch_zstd:compress(Term, Level); compress(Term, snappy) -> Bin = ?term_to_bin(Term), try @@ -63,6 +86,8 @@ compress(Term, snappy) -> decompress(<<?SNAPPY_PREFIX, Rest/binary>>) -> {ok, TermBin} = snappy:decompress(Rest), binary_to_term(TermBin); +decompress(<<?ZSTD_MAGIC, _/binary>> = Bin) -> + couch_zstd:decompress(Bin); decompress(<<?TERM_PREFIX, _/binary>> = Bin) -> binary_to_term(Bin); decompress(_) -> @@ -74,6 +99,10 @@ is_compressed(<<?COMPRESSED_TERM_PREFIX, _/binary>>, {deflate, _Level}) -> true; is_compressed(<<?COMPRESSED_TERM_PREFIX, _/binary>>, _Method) -> false; +is_compressed(<<?ZSTD_MAGIC, _/binary>>, {zstd, _Level}) -> + true; +is_compressed(<<?ZSTD_MAGIC, _/binary>>, _Method) -> + false; is_compressed(<<?TERM_PREFIX, _/binary>>, Method) -> Method =:= none; is_compressed(Term, _Method) when not is_binary(Term) -> @@ -84,6 +113,8 @@ is_compressed(_, _) -> uncompressed_size(<<?SNAPPY_PREFIX, Rest/binary>>) -> {ok, Size} = snappy:uncompressed_length(Rest), Size; +uncompressed_size(<<?ZSTD_MAGIC, _/binary>> = Bin) -> + couch_zstd:uncompressed_size(Bin); uncompressed_size(<<?COMPRESSED_TERM_PREFIX, Size:32, _/binary>> = _Bin) -> % See http://erlang.org/doc/apps/erts/erl_ext_dist.html % The uncompressed binary would be encoded with <<131, Rest/binary>> diff --git a/src/couch/src/couch_zstd.erl b/src/couch/src/couch_zstd.erl new file mode 100644 index 000000000..8c8b7819b --- /dev/null +++ b/src/couch/src/couch_zstd.erl @@ -0,0 +1,56 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +% Thin wrapper for Erlang's zstd (available in OTP 28+) + +-module(couch_zstd). + +-export([ + available/0, + compress/2, + decompress/1, + uncompressed_size/1 +]). + +-if(?OTP_RELEASE >= 28). + +available() -> + true. + +compress(Term, Level) when is_integer(Level) -> + % Since compress/2 can take an iovec and, term_to_iovec/2 exists, plug into + % the other for a potential performance win + Iovec = term_to_iovec(Term, [{minor_version, 1}]), + iolist_to_binary(zstd:compress(Iovec, #{compressionLevel => Level})). + +decompress(Bin) when is_binary(Bin) -> + binary_to_term(iolist_to_binary(zstd:decompress(Bin))). + +uncompressed_size(Bin) when is_binary(Bin) -> + {ok, #{frameContentSize := Size}} = zstd:get_frame_header(Bin), + Size. + +-else. + +available() -> + false. + +compress(_Bin, _Level) -> + erlang:error(zstd_unavailable). + +decompress(_Bin) -> + erlang:error(zstd_unavailable). + +uncompressed_size(_Bin) -> + erlang:error(zstd_unavailable). + +-endif. diff --git a/src/couch/test/eunit/couch_compress_tests.erl b/src/couch/test/eunit/couch_compress_tests.erl index 3d11be41a..67c13378c 100644 --- a/src/couch/test/eunit/couch_compress_tests.erl +++ b/src/couch/test/eunit/couch_compress_tests.erl @@ -37,6 +37,7 @@ -define(DEFLATE_COMPRESSION, {deflate, 9}). -define(DEFLATE_COMPRESSION_ZERO, {deflate, 0}). +-define(ZSTD_COMPRESSION, {zstd, 3}). couch_compress_test_() -> { @@ -46,7 +47,8 @@ couch_compress_test_() -> fun t_decompress/0, fun t_recompress/0, fun t_is_compressed/0, - fun t_uncompressed_size/0 + fun t_uncompressed_size/0, + fun t_zstd/0 ] }. @@ -126,3 +128,38 @@ t_uncompressed_size() -> ), ?assertError(invalid_compression, couch_compress:uncompressed_size(?CORRUPT)). + +t_zstd() -> + case couch_zstd:available() of + false -> + % Only available on 28+ + ?assertNot(couch_zstd:available()); + true -> + Z = couch_compress:compress(?TERM, ?ZSTD_COMPRESSION), + ?assertMatch(<<16#28, 16#B5, 16#2F, 16#FD, _/binary>>, Z), + ?assert(bit_size(Z) < bit_size(?NONE)), + ?assertEqual(?TERM, couch_compress:decompress(Z)), + + % Method checks + ?assert(couch_compress:is_compressed(Z, ?ZSTD_COMPRESSION)), + ?assert(couch_compress:is_compressed(Z, {zstd, 19})), + ?assertNot(couch_compress:is_compressed(Z, snappy)), + ?assertNot(couch_compress:is_compressed(Z, none)), + ?assertNot(couch_compress:is_compressed(Z, ?DEFLATE_COMPRESSION)), + ?assertNot(couch_compress:is_compressed(?NONE, ?ZSTD_COMPRESSION)), + ?assertNot(couch_compress:is_compressed(?SNAPPY, ?ZSTD_COMPRESSION)), + ?assertNot(couch_compress:is_compressed(?DEFLATE, ?ZSTD_COMPRESSION)), + + ?assertEqual(49, couch_compress:uncompressed_size(Z)), + + % Already compressed checks + ?assertEqual(Z, couch_compress:compress(Z, ?ZSTD_COMPRESSION)), + Snappy = couch_compress:compress(?SNAPPY, ?ZSTD_COMPRESSION), + ?assert(couch_compress:is_compressed(Snappy, ?ZSTD_COMPRESSION)), + ?assertEqual(?TERM, couch_compress:decompress(Snappy)), + Deflate = couch_compress:compress(Z, ?DEFLATE_COMPRESSION), + ?assert(couch_compress:is_compressed(Deflate, ?DEFLATE_COMPRESSION)), + ?assertEqual(?TERM, couch_compress:decompress(Deflate)), + None = couch_compress:compress(Z, none), + ?assert(couch_compress:is_compressed(None, none)) + end. diff --git a/src/couch/test/eunit/couchdb_file_compression_tests.erl b/src/couch/test/eunit/couchdb_file_compression_tests.erl index c3ec4174a..f03e485cb 100644 --- a/src/couch/test/eunit/couchdb_file_compression_tests.erl +++ b/src/couch/test/eunit/couchdb_file_compression_tests.erl @@ -44,6 +44,7 @@ couch_file_compression_test_() -> ?TDEF_FE(should_use_deflate_1, ?TIMEOUT), ?TDEF_FE(should_use_deflate_9, ?TIMEOUT), ?TDEF_FE(should_use_snappy, ?TIMEOUT), + ?TDEF_FE(should_use_zstd, ?TIMEOUT), ?TDEF_FE(should_compare_compression_methods, ?TIMEOUT) ]} } @@ -104,6 +105,11 @@ should_use_snappy(DbName) -> compact_db(DbName), compact_view(DbName). +should_use_zstd(DbName) -> + config:set("couchdb", "file_compression", "zstd_3", false), + compact_db(DbName), + compact_view(DbName). + should_compare_compression_methods(DbName) -> config:set("couchdb", "file_compression", "none", false), ExternalSizePreCompact = db_external_size(DbName),
