This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch improve-term-parsing in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5b2d103b9509a30c0526b3cbe9c7be665275a795 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Sat Nov 14 11:47:22 2026 -0500 Tighten options and term decoding Improve replicator options and term decoding. Add some limits and for options, since we already have an accepted allow-list that only takes string, booleans and numbers just use a plain regex for it. --- src/couch/src/couch_util.erl | 22 ++++ src/couch/test/eunit/couch_util_tests.erl | 10 ++ .../src/couch_replicator_parse.erl | 126 +++++++++++++++++---- src/dreyfus/src/dreyfus_bookmark.erl | 2 +- src/fabric/src/fabric_view_changes.erl | 2 +- src/mango/src/mango_json_bookmark.erl | 2 +- 6 files changed, 139 insertions(+), 25 deletions(-) diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl index 6ac8a47ea..b25b1458c 100644 --- a/src/couch/src/couch_util.erl +++ b/src/couch/src/couch_util.erl @@ -48,6 +48,7 @@ -export([new_set/0, set_from_list/1]). -export([hibernate_after/1]). -export([gunzip_with_limit/1, gunzip_with_limit/2]). +-export([binary_to_term_safe/1, binary_to_term_safe/2]). -include_lib("couch/include/couch_db.hrl"). @@ -68,6 +69,9 @@ -define(DEFAULT_HIBERNATE_AFTER, 5000). +% Max uncompressed size param terms +-define(TERM_MAX_SIZE, 1048576). + priv_dir() -> case code:priv_dir(couch) of {error, bad_name} -> @@ -826,6 +830,24 @@ hibernate_after(Module) when is_atom(Module) -> [{hibernate_after, Timeout}] end. +binary_to_term_safe(Bin) -> + binary_to_term_safe(Bin, ?TERM_MAX_SIZE). + +binary_to_term_safe(Bin, MaxSize) when is_integer(MaxSize), MaxSize >= 0 -> + case ext_term_size(Bin) of + Size when Size > MaxSize -> error({term_too_large, Size}); + _ -> binary_to_term(Bin, [safe]) + end. + +ext_term_size(<<131, 80, Size:32, _/binary>> = Bin) -> + % See https://www.erlang.org/doc/apps/erts/erl_ext_dist.html + max(Size, byte_size(Bin)); +ext_term_size(<<Bin/binary>>) -> + byte_size(Bin); +ext_term_size(_NotBinary) -> + % we'll get a badarg in binary_to_term for it + 0. + gunzip_with_limit(Data) -> Limit = chttpd_util:get_chttpd_config_integer("max_http_request_size", 4294967296), gunzip_with_limit(Data, Limit). diff --git a/src/couch/test/eunit/couch_util_tests.erl b/src/couch/test/eunit/couch_util_tests.erl index ac8b5119c..fcafdaaa3 100644 --- a/src/couch/test/eunit/couch_util_tests.erl +++ b/src/couch/test/eunit/couch_util_tests.erl @@ -316,3 +316,13 @@ url_strip_password_test() -> Cases ), ?assertEqual(undefined, couch_util:url_strip_password(undefined)). + +binary_to_term_safe_too_large_test() -> + % Not compressed + Bin = term_to_binary(binary:copy(<<0>>, 1000)), + ?assertEqual(1000, byte_size(couch_util:binary_to_term_safe(Bin, byte_size(Bin)))), + ?assertError({term_too_large, _}, couch_util:binary_to_term_safe(Bin, byte_size(Bin) - 1)), + % Compressed + Compressed = term_to_binary(binary:copy(<<0>>, 100000), [compressed]), + ?assertEqual(100000, byte_size(couch_util:binary_to_term_safe(Compressed, 100005))), + ?assertError({term_too_large, 100005}, couch_util:binary_to_term_safe(Compressed, 100004)). diff --git a/src/couch_replicator/src/couch_replicator_parse.erl b/src/couch_replicator/src/couch_replicator_parse.erl index 985ca911b..23486b3e6 100644 --- a/src/couch_replicator/src/couch_replicator_parse.erl +++ b/src/couch_replicator/src/couch_replicator_parse.erl @@ -38,6 +38,9 @@ -define(VALID_IBROWSE_OPTS, [ prefer_ipv6 ]). +% {K, V} option tuples. K is binary and V = true | false | 0..9 +-define(OPT_RE, "\\{\\s*([a-z][a-z0-9_]*)\\s*,\\s*(true|false|[0-9]{1,9})\\s*\\}"). +-define(OPT_MAX, 1024). -define(VALID_PROTOCOLS, #{ endpoint => [http, https], proxy => [http, https, socks5] @@ -297,34 +300,45 @@ cfg_atoms(Cfg, Default) -> end. cfg_sock_opts() -> - CfgTerm = cfg("socket_options"), - parse_opts(CfgTerm, ?DEFAULT_SOCK_OPTS, ?VALID_SOCK_OPTS). + cfg_opts("socket_options", ?DEFAULT_SOCK_OPTS, ?VALID_SOCK_OPTS). cfg_ibrowse_opts() -> - CfgTerm = cfg("ibrowse_options"), - parse_opts(CfgTerm, ?DEFAULT_IBROWSE_OPTS, ?VALID_IBROWSE_OPTS). + cfg_opts("ibrowse_options", ?DEFAULT_IBROWSE_OPTS, ?VALID_IBROWSE_OPTS). cfg(Var) -> config:get("replicator", Var). -parse_opts(undefined, Defaults, _) -> - Defaults; -parse_opts(Term, _Defaults, ValidOpts) -> - SocketOptions = - case couch_util:parse_term(Term) of - {ok, Opts} -> Opts; - {error, _Error} -> [] - end, - Fun = fun({K, _}) -> lists:member(K, ValidOpts) end, - lists:filtermap(Fun, SocketOptions). +cfg_opts(Var, Defaults, ValidOpts) -> + case cfg(Var) of + undefined -> Defaults; + Val -> parse_opts(list_to_binary(Val), ValidOpts) + end. -sock_opts(CfgTerm) -> +sock_opts(Val) -> ValidOpts = cfg_atoms("valid_socket_options", ?VALID_SOCK_OPTS), - parse_opts(CfgTerm, ?DEFAULT_SOCK_OPTS, ValidOpts). + parse_opts(Val, ValidOpts). -ibrowse_opts(CfgTerm) -> +ibrowse_opts(Val) -> ValidOpts = cfg_atoms("valid_ibrowse_options", ?VALID_IBROWSE_OPTS), - parse_opts(CfgTerm, ?DEFAULT_IBROWSE_OPTS, ValidOpts). + parse_opts(Val, ValidOpts). + +parse_opts(Val, ValidOpts) when is_binary(Val), byte_size(Val) =< ?OPT_MAX -> + Names = #{atom_to_binary(Name) => Name || Name <- ValidOpts}, + case re:run(Val, ?OPT_RE, [global, {capture, all_but_first, binary}]) of + {match, Pairs} -> + [{map_get(K, Names), opt_val(V)} || [K, V] <- Pairs, is_map_key(K, Names)]; + nomatch -> + [] + end; +parse_opts(_, _ValidOpts) -> + []. + +opt_val(~"true") -> + true; +opt_val(~"false") -> + false; +opt_val(<<Digits/binary>>) -> + binary_to_integer(Digits). -spec convert_options([_]) -> [_]. convert_options([]) -> @@ -384,9 +398,9 @@ convert_options([{<<"connection_timeout">>, V} | R]) -> [{connection_timeout, couch_util:to_integer(V)} | convert_options(R)]; convert_options([{<<"retries_per_request">>, V} | R]) -> [{retries, couch_util:to_integer(V)} | convert_options(R)]; -convert_options([{<<"socket_options">>, V} | R]) -> +convert_options([{<<"socket_options">>, V} | R]) when is_binary(V) -> [{socket_options, sock_opts(V)} | convert_options(R)]; -convert_options([{<<"ibrowse_options">>, V} | R]) -> +convert_options([{<<"ibrowse_options">>, V} | R]) when is_binary(V) -> [{ibrowse_options, ibrowse_opts(V)} | convert_options(R)]; convert_options([{<<"since_seq">>, V} | R]) -> [{since_seq, V} | convert_options(R)]; @@ -642,7 +656,10 @@ local_replication_endpoint_error_test_() -> ?TDEF_FE(t_parse_proxy_invalid_protocol), ?TDEF_FE(t_parse_sock_opts), ?TDEF_FE(t_parse_ibrowse_opts), - ?TDEF_FE(t_parse_opts_invalid) + ?TDEF_FE(t_parse_opts_invalid), + ?TDEF_FE(t_parse_opts), + ?TDEF_FE(t_parse_opts_max_size), + ?TDEF_FE(t_parse_opts_no_new_atoms) ] }. @@ -850,6 +867,71 @@ t_parse_ibrowse_opts(_) -> ). t_parse_opts_invalid(_) -> - ?assertEqual([], parse_opts(<<"<}garbage]][">>, [], [])). + Ignored = [ + ~"", + ~"randomjunk", + ~"<}potato]][", + ~"keepalive", + ~"[keepalive, nodelay]", + ~"[{keepalive}]", + ~"[{keepalive, true, true}]", + ~"[{'keepalive', true}]", + ~"[{keepalive, 'true'}]", + ~"[{keepalive, \"true\"}]", + ~"[{recbuf, -1}]", + ~"[{recbuf, 1.5}]", + ~"[{recbuf, 16#ff}]", + ~"[{recbuf, 1_000}]", + ~"[{recbuf, [1]}]", + ~"[{recbuf, 1234567890}]" + ], + [?assertEqual([], sock_opts(Val), Val) || Val <- Ignored]. + +t_parse_opts(_) -> + ?assertEqual([], sock_opts(~"[]")), + ?assertEqual( + [{keepalive, true}, {nodelay, false}], + sock_opts(~"[{keepalive, true}, {nodelay, false}]") + ), + ?assertEqual( + [{keepalive, false}, {recbuf, 65536}], + sock_opts(~"[{keepalive, false},{ recbuf,65536 } ]") + ), + % no binaries + ?assertEqual([], sock_opts(~"[{recbuf, <<1>>}]")). + +t_parse_opts_max_size(_) -> + Opts = ~"[{nodelay, true}, {prefer_ipv6, true}]", + Pad = fun(Sz) -> <<Opts/binary, (binary:copy(~" ", Sz - byte_size(Opts)))/binary>> end, + ?assertEqual([{nodelay, true}], sock_opts(Pad(?OPT_MAX))), + ?assertEqual([{prefer_ipv6, true}], ibrowse_opts(Pad(?OPT_MAX))), + ?assertEqual([], sock_opts(Pad(?OPT_MAX + 1))), + ?assertEqual([], ibrowse_opts(Pad(?OPT_MAX + 1))). + +t_parse_opts_no_new_atoms(_) -> + [N1, N2, N3] = [new_name() || _ <- lists:seq(1, 3)], + F = fun(Fmt, Args) -> iolist_to_binary(io_lib:format(Fmt, Args)) end, + ?assertEqual([], sock_opts(F("[~s, '~s']", [N1, N2]))), + ?assertEqual([], sock_opts(F("[{~s, 1}, {'~s', true}]", [N1, N2]))), + ?assertEqual([], ibrowse_opts(F("[{~s, 1}, {'~s', true}]", [N1, N2]))), + ?assertEqual([], sock_opts(F("[{keepalive, ~s}, {recbuf, ['~s']}]", [N3, N3]))), + RepDoc = + {[ + {~"source", ~"http://a"}, + {~"target", ~"http://b"}, + {~"socket_options", F("[{nodelay, true}, {~s, 1}]", [N1])}, + {~"ibrowse_options", F("[~s]", [N2])} + ]}, + #rep{options = Options} = parse_rep_doc_without_id(RepDoc), + ?assertEqual([{nodelay, true}], get_value(socket_options, Options)), + ?assertEqual([], get_value(ibrowse_options, Options)), + % We never created those new atoms + ?assertError(badarg, binary_to_existing_atom(N1)), + ?assertError(badarg, binary_to_existing_atom(N2)), + ?assertError(badarg, binary_to_existing_atom(N3)). + +new_name() -> + Uniq = integer_to_binary(erlang:unique_integer([positive])), + <<"anatom_", Uniq/binary>>. -endif. diff --git a/src/dreyfus/src/dreyfus_bookmark.erl b/src/dreyfus/src/dreyfus_bookmark.erl index 268a6b845..145dda0b7 100644 --- a/src/dreyfus/src/dreyfus_bookmark.erl +++ b/src/dreyfus/src/dreyfus_bookmark.erl @@ -67,7 +67,7 @@ unpack(DbName, Packed) when is_binary(Packed) -> ). unpack_entries(Packed) -> - case binary_to_term(couch_util:decodeBase64Url(Packed), [safe]) of + case couch_util:binary_to_term_safe(couch_util:decodeBase64Url(Packed)) of Entries when is_list(Entries) -> lists:foreach(fun validate_entry/1, Entries), Entries; diff --git a/src/fabric/src/fabric_view_changes.erl b/src/fabric/src/fabric_view_changes.erl index f97475b8a..f79f6219c 100644 --- a/src/fabric/src/fabric_view_changes.erl +++ b/src/fabric/src/fabric_view_changes.erl @@ -489,7 +489,7 @@ unpack_seq_term(Opaque) -> % Looks like a valid term_to_binary at least SeqList = try - binary_to_term(Bin, [safe]) + couch_util:binary_to_term_safe(Bin) catch error:badarg -> error(?BADSEQ) end, diff --git a/src/mango/src/mango_json_bookmark.erl b/src/mango/src/mango_json_bookmark.erl index 8446e0c8a..9ddf194f3 100644 --- a/src/mango/src/mango_json_bookmark.erl +++ b/src/mango/src/mango_json_bookmark.erl @@ -52,7 +52,7 @@ unpack(nil) -> nil; unpack(Packed) -> try - Bookmark = binary_to_term(couch_util:decodeBase64Url(Packed), [safe]), + Bookmark = couch_util:binary_to_term_safe(couch_util:decodeBase64Url(Packed)), verify(Bookmark) catch _:_ ->
