This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch 3.5.3-rc2-fixes in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 16f0772273987a6402ad6a78fe2e2d69726ba2f3 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Tue Oct 6 12:30:41 2026 -0400 Strip more replicator creds Fix non-owner readers seeing credentials in a shared _replicator db. Also, while at it, fix stripping of creds for logging and API endpoints returning replication stats (cherry picked from commit afb135c26f6346f3ac484e6a7a02506a10762f19) --- src/couch/src/couch_util.erl | 6 +- src/couch/test/eunit/couch_util_tests.erl | 30 +++ src/couch_replicator/src/couch_replicator_docs.erl | 207 ++++++++++++++++++--- .../src/couch_replicator_scheduler_job.erl | 118 +++++++++++- 4 files changed, 335 insertions(+), 26 deletions(-) diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl index 7a82888f2..6ac8a47ea 100644 --- a/src/couch/src/couch_util.erl +++ b/src/couch/src/couch_util.erl @@ -517,10 +517,12 @@ reorder_results(Keys, SortedResults, Default) -> [maps:get(Key, Map, Default) || Key <- Keys]. url_strip_password(Url) when is_list(Url) -> + % \1: any scheme any case + % \2: username, possibly empty re:replace( Url, - "(http|https|socks5)://([^:]+):[^@]+@(.*)$", - "\\1://\\2:*****@\\3", + "^(\\s*[a-zA-Z][a-zA-Z0-9+.-]*://)([^:/?#]*):[^/?#]*@", + "\\1\\2:*****@", [{return, list}] ); url_strip_password(Other) -> diff --git a/src/couch/test/eunit/couch_util_tests.erl b/src/couch/test/eunit/couch_util_tests.erl index ff40a6fdd..ac8b5119c 100644 --- a/src/couch/test/eunit/couch_util_tests.erl +++ b/src/couch/test/eunit/couch_util_tests.erl @@ -286,3 +286,33 @@ wait_hibernate(Pid) -> end end, test_util:wait(WaitFun). + +url_strip_password_test() -> + Cases = [ + {"http://h/db", "http://h/db"}, + {"http://h:15984/db", "http://h:15984/db"}, + {"http://u@h/db", "http://u@h/db"}, + {"http://u:p@h/db", "http://u:*****@h/db"}, + {"https://u:p@h:5984/db?a=b", "https://u:*****@h:5984/db?a=b"}, + {"socks5://u:p@h:1081", "socks5://u:*****@h:1081"}, + {"HTTPS://u:p@h/db", "HTTPS://u:*****@h/db"}, + {"https://:p@h/db", "https://:*****@h/db"}, + {"https://u:p@ss@h/db", "https://u:*****@h/db"}, + {"https://u:p%40ss@h/db", "https://u:*****@h/db"}, + {"http://u:p@[::1]:25984/db", "http://u:*****@[::1]:25984/db"}, + {"http://u:p@h/d@b", "http://u:*****@h/d@b"}, + {"http://h/d:x@b", "http://h/d:x@b"}, + {"http://h:35984?x=a@b", "http://h:35984?x=a@b"}, + {"https://[email protected]:p@h/db", "https://[email protected]:*****@h/db"}, + {"https://[email protected]:p@ss@h/db", "https://[email protected]:*****@h/db"}, + {"https://[email protected]@h/db", "https://[email protected]@h/db"}, + {"http://u@h:15984/db", "http://u@h:15984/db"}, + {"http://[::1]:25984/d@b", "http://[::1]:25984/d@b"} + ], + lists:foreach( + fun({Url, Expect}) -> + ?assertEqual(Expect, couch_util:url_strip_password(Url)) + end, + Cases + ), + ?assertEqual(undefined, couch_util:url_strip_password(undefined)). diff --git a/src/couch_replicator/src/couch_replicator_docs.erl b/src/couch_replicator/src/couch_replicator_docs.erl index 8f134aaab..1df692080 100644 --- a/src/couch_replicator/src/couch_replicator_docs.erl +++ b/src/couch_replicator/src/couch_replicator_docs.erl @@ -295,10 +295,7 @@ after_doc_read(#doc{body = {Body}} = Doc, Db) -> % replicator app: this is another user reading it. Strip % the creds and make up a fake "rev" since we're returning % a technically non-existent document body - Source = strip_credentials(get_value(<<"source">>, Body)), - Target = strip_credentials(get_value(<<"target">>, Body)), - NewBody0 = ?replace(Body, <<"source">>, Source), - NewBody = ?replace(NewBody0, <<"target">>, Target), + NewBody = strip_body_credentials(Body), #doc{revs = {Pos, [_ | Revs]}} = Doc, NewDoc = Doc#doc{body = {NewBody}, revs = {Pos - 1, Revs}}, NewRevId = couch_db:new_revid(NewDoc), @@ -306,25 +303,61 @@ after_doc_read(#doc{body = {Body}} = Doc, Db) -> end end. +-define(CREDS, [~"source", ~"target", ~"proxy", ~"source_proxy", ~"target_proxy"]). + +strip_body_credentials(Body) -> + lists:map( + fun({K, V}) -> + case lists:member(K, ?CREDS) of + true -> {K, strip_credentials(V)}; + false -> {K, V} + end + end, + Body + ). + -spec strip_credentials - (undefined) -> undefined; - (binary()) -> binary(); - ({[_]}) -> {[_]}. -strip_credentials(undefined) -> - undefined; + (binary()) -> binary() | null; + ({[_]}) -> {[_]}; + (number() | boolean() | null | list()) -> null. strip_credentials(Url) when is_binary(Url) -> - re:replace( - Url, - "http(s)?://(?:[^:]+):[^@]+@(.*)$", - "http\\1://\\2", - [{return, binary}] - ); -strip_credentials({Props0}) -> + strip_url_userinfo(Url); +strip_credentials({Props0}) when is_list(Props0) -> Props1 = lists:keydelete(<<"headers">>, 1, Props0), % Strip "auth" just like headers, for replication plugins it can be a place % to stash credential that are not necessarily in headers Props2 = lists:keydelete(<<"auth">>, 1, Props1), - {Props2}. + % Object form endpoints may have creds in "url" + Props3 = + case lists:keyfind(~"url", 1, Props2) of + {~"url", Url} -> ?replace(Props2, ~"url", strip_credentials(Url)); + false -> Props2 + end, + {Props3}; +strip_credentials(_Other) -> + % Failed or broken replication docs. Just hide the whole thing as might + % still contain mistyped creds. + null. + +% Remove userinfo from url. Match the parser with the replicator. If we can't +% parse remove everything between scheme and last "@" +% +strip_url_userinfo(Url) when is_binary(Url) -> + case uri_string:parse(Url) of + #{userinfo := _} = UriMap -> + case uri_string:recompose(maps:remove(userinfo, UriMap)) of + <<_/binary>> = NoCreds -> NoCreds; + {error, _, _} -> null + end; + #{} -> + Url; + {error, _, _} -> + % \1 is scheme in any case, maybe with leading spaces. Use dotall + % to match even newlines in case passwords have those too somehow + re:replace(Url, "^(\\s*[a-zA-Z][a-zA-Z0-9+.-]*://).*@", "\\1", [ + dotall, {return, binary} + ]) + end. error_reason({shutdown, Error}) -> error_reason(Error); @@ -356,10 +389,6 @@ check_strip_credentials_test() -> [ ?assertEqual(Expected, strip_credentials(Body)) || {Expected, Body} <- [ - { - undefined, - undefined - }, { <<"https://remote_server/database">>, <<"https://foo:bar@remote_server/database">> @@ -379,10 +408,95 @@ check_strip_credentials_test() -> { {[{<<"_id">>, <<"foo">>}]}, {[{<<"_id">>, <<"foo">>}, {<<"auth">>, <<"pluginsecret">>}]} + }, + { + {[{~"url", ~"https://remote_server/database"}]}, + {[{~"url", ~"https://foo:bar@remote_server/database"}]} + }, + { + {[{~"url", ~"http://remote_server/database"}]}, + {[ + {~"url", ~"http://foo:bar@remote_server/database"}, + {~"headers", ~"bar"}, + {~"auth", ~"pluginsecret"} + ]} + }, + { + {[{~"url", ~"https://remote_server/database"}]}, + {[{~"url", ~"https://foo:p%40ss%3Aw@remote_server/database"}]} + }, + { + ~"HTTPS://remote_server/database", + ~"HTTPS://foo:bar@remote_server/database" + }, + { + ~"https://remote_server/database", + ~"https://:bar@remote_server/database" + }, + { + ~"https://remote_server/database", + ~"https://foo@remote_server/database" + }, + { + ~"https://remote_server/database", + ~"https://foo:b@r@remote_server/database" + }, + { + ~"https://remote_server/database", + ~"https://foo:b/r@remote_server/database" + }, + { + ~"http://[::1]:5984/db?a=b", + ~"http://foo:bar@[::1]:5984/db?a=b" + }, + { + ~"http://remote_server/d@b", + ~"http://foo:bar@remote_server/d@b" + }, + { + ~"http://remote_server/database", + ~"http://remote_server/database" + }, + { + ~"localdb", + ~"localdb" + }, + { + null, + [~"https://foo:bar@remote_server/database"] + }, + { + null, + 42 } ] ]. +check_strip_body_credentials_test() -> + Body = [ + {~"_id", ~"foo"}, + {~"source", {[{~"url", ~"https://u:p1@s/db"}]}}, + {~"target", ~"https://u:p2@t/db"}, + {~"proxy", ~"http://u:p3@proxy:3128"}, + {~"source_proxy", ~"socks5://u:p4@sproxy:1080"}, + {~"target_proxy", ~"http://u:p5@tproxy:3128"}, + {~"other", ~"https://u:p6@x/db"} + ], + ?assertEqual( + [ + {~"_id", ~"foo"}, + {~"source", {[{~"url", ~"https://s/db"}]}}, + {~"target", ~"https://t/db"}, + {~"proxy", ~"http://proxy:3128"}, + {~"source_proxy", ~"socks5://sproxy:1080"}, + {~"target_proxy", ~"http://tproxy:3128"}, + {~"other", ~"https://u:p6@x/db"} + ], + strip_body_credentials(Body) + ), + % Missing fields are not added + ?assertEqual([{~"_id", ~"foo"}], strip_body_credentials([{~"_id", ~"foo"}])). + setup() -> TmpDbName = ?tempdb(), DbName = <<TmpDbName/binary, "/_replicator">>, @@ -450,7 +564,9 @@ replicator_can_update_docs_test_() -> ?TDEF_FE(t_after_doc_read_as_replicator), ?TDEF_FE(t_after_doc_read_internal_replicator), ?TDEF_FE(t_after_doc_read_matching_owner), - ?TDEF_FE(t_after_doc_read_not_matching_owner) + ?TDEF_FE(t_after_doc_read_not_matching_owner), + ?TDEF_FE(t_after_doc_read_not_matching_owner_object_endpoints), + ?TDEF_FE(t_after_doc_read_not_matching_owner_proxies) ] } }. @@ -605,6 +721,53 @@ t_after_doc_read_not_matching_owner(DbName) -> }, ?assertEqual(StrippedMap, Map1). +t_after_doc_read_not_matching_owner_object_endpoints(DbName) -> + DocId = ~"doc1", + Map = #{ + ~"source" => #{~"url" => ~"https://user1:pass1@localhost:5984/db1"}, + ~"target" => #{ + ~"url" => ~"https://user2:pass2@localhost:5984/db2", + ~"headers" => #{~"Authorization" => ~"Bearer secret"}, + ~"auth" => #{ + ~"basic" => #{~"username" => ~"u", ~"password" => ~"p"} + } + }, + ?OWNER => ~"o1" + }, + {ok, _} = write_doc(DbName, DocId, Map), + Ctx = {user_ctx, #user_ctx{name = ~"o2", roles = [~"tomato", ~"potato"]}}, + Map1 = read_doc(DbName, DocId, Ctx), + StrippedMap = #{ + ~"source" => #{~"url" => ~"https://localhost:5984/db1"}, + ~"target" => #{~"url" => ~"https://localhost:5984/db2"}, + ?OWNER => ~"o1" + }, + ?assertEqual(StrippedMap, Map1). + +t_after_doc_read_not_matching_owner_proxies(DbName) -> + DocId = ~"doc1", + Map = #{ + ~"source" => ~"https://localhost:5984/db1", + ~"target" => ~"https://localhost:5984/db2", + ~"source_proxy" => ~"http://user1:pass1@proxy1:3128", + ~"target_proxy" => ~"socks5://user2:pass2@proxy2:1080", + ?OWNER => ~"o1" + }, + {ok, _} = write_doc(DbName, DocId, Map), + Ctx = {user_ctx, #user_ctx{name = ~"o2", roles = [~"tomato", ~"potato"]}}, + Map1 = read_doc(DbName, DocId, Ctx), + StrippedMap = #{ + ~"source" => ~"https://localhost:5984/db1", + ~"target" => ~"https://localhost:5984/db2", + ~"source_proxy" => ~"http://proxy1:3128", + ~"target_proxy" => ~"socks5://proxy2:1080", + ?OWNER => ~"o1" + }, + ?assertEqual(StrippedMap, Map1), + % Owner still sees everything + OwnerCtx = {user_ctx, #user_ctx{name = ~"o1", roles = []}}, + ?assertEqual(Map, read_doc(DbName, DocId, OwnerCtx)). + ejson_from_map(#{} = Map) -> ?JSON_DECODE(?JSON_ENCODE(Map)). diff --git a/src/couch_replicator/src/couch_replicator_scheduler_job.erl b/src/couch_replicator/src/couch_replicator_scheduler_job.erl index 55a28ad7e..f795917bc 100644 --- a/src/couch_replicator/src/couch_replicator_scheduler_job.erl +++ b/src/couch_replicator/src/couch_replicator_scheduler_job.erl @@ -544,10 +544,48 @@ headers_strip_creds([{Key, Value0} | Rest], Acc) -> end, headers_strip_creds(Rest, [{Key, Value} | Acc]). -httpdb_strip_creds(#httpdb{url = Url, headers = Headers} = HttpDb) -> +% normalize_basic_auth moves basic user:pass combo to an auth props object +auth_props_strip_creds(Props) -> + lists:map( + fun + ({~"username", User}) when is_binary(User) -> {~"username", User}; + ({Key, {Nested}}) when is_list(Nested) -> {Key, {auth_props_strip_creds(Nested)}}; + ({Key, _}) -> {Key, ~"*****"}; + (_) -> ~"*****" + end, + Props + ). + +% ibrowse opts can have cred too for proxies and ssl options +ibrowse_options_strip_creds(Options) -> + lists:map( + fun + ({proxy_password, _}) -> + {proxy_password, "*****"}; + ({socks5_password, _}) -> + {socks5_password, "*****"}; + ({ssl_options, SslOpts}) when is_list(SslOpts) -> + {ssl_options, lists:keyreplace(password, 1, SslOpts, {password, "*****"})}; + (Opt) -> + Opt + end, + Options + ). + +httpdb_strip_creds(#httpdb{} = HttpDb) -> + #httpdb{ + url = Url, + headers = Headers, + proxy_url = ProxyUrl, + auth_props = AuthProps, + ibrowse_options = IbrowseOptions + } = HttpDb, HttpDb#httpdb{ url = couch_util:url_strip_password(Url), - headers = headers_strip_creds(Headers, []) + headers = headers_strip_creds(Headers, []), + proxy_url = couch_util:url_strip_password(ProxyUrl), + auth_props = auth_props_strip_creds(AuthProps), + ibrowse_options = ibrowse_options_strip_creds(IbrowseOptions) }; httpdb_strip_creds(LocalDb) -> LocalDb. @@ -1274,4 +1312,80 @@ t_scheduler_job_format_status(_) -> ?assertEqual(<<"4">>, maps:get(current_through_seq, State)), ?assertEqual(<<"5">>, maps:get(highest_seq_done, State)). +httpdb_rec_strip_creds_test_() -> + { + foreach, + fun meck_config/0, + fun(_) -> meck:unload() end, + [ + ?TDEF_FE(t_httpdb_strip_creds), + ?TDEF_FE(t_httpdb_strip_creds_socks5) + ] + }. + +t_httpdb_strip_creds(_) -> + Endpoint = + {[ + {~"url", ~"http://u:urlpass@h1/d1"}, + {~"headers", {[{~"Authorization", ~"Bearer hidethebear"}]}} + ]}, + Proxy = ~"http://pu:psecret@myproxy:8080", + HttpDb = couch_replicator_parse:parse_rep_db(Endpoint, Proxy, []), + HttpDb1 = httpdb_strip_creds(HttpDb), + % Doesn't show up if we dump the data structure to the log + ToStr = lists:flatten(io_lib:format("~p", [HttpDb1])), + [ + ?assertEqual(nomatch, string:find(ToStr, S)) + || S <- ["urlpass", "hidethebear", "psecret"] + ], + ?assertEqual("http://pu:*****@myproxy:8080", HttpDb1#httpdb.proxy_url), + ?assertEqual({"u", "*****"}, couch_replicator_utils:get_basic_auth_creds(HttpDb1)). + +t_httpdb_strip_creds_socks5(_) -> + % The proxy password can gets stuck into the ibrowse options so we check + % it's cleaned up there too + Proxy = ~"socks5://pu:psecret@myproxy:2080", + HttpDb = couch_replicator_parse:parse_rep_db(~"http://h1/d1", Proxy, []), + HttpDb1 = httpdb_strip_creds(HttpDb), + ToStr = lists:flatten(io_lib:format("~p", [HttpDb1])), + ?assertEqual(nomatch, string:find(ToStr, "psecret")), + ?assertEqual("*****", couch_util:get_value(socks5_password, HttpDb1#httpdb.ibrowse_options)). + +httpdb_strip_creds_basic_auth_test() -> + % normalize_basic_auth/1 moves the creds from the url/headers to auth_props + % see if that gets cleaned up + Header = {"Authorization", "Basic " ++ base64:encode_to_string("user2:secret2")}, + Basic = {[{~"username", ~"user3"}, {~"password", ~"secret3"}]}, + lists:foreach( + fun({User, Pass, HttpDb0}) -> + HttpDb = couch_replicator_utils:normalize_basic_auth(HttpDb0), + ?assertEqual({User, Pass}, couch_replicator_utils:get_basic_auth_creds(HttpDb)), + HttpDb1 = httpdb_strip_creds(HttpDb), + ToStr = lists:flatten(io_lib:format("~p", [HttpDb1])), + ?assertEqual(nomatch, string:find(ToStr, Pass)) + end, + [ + {"user1", "secret1", #httpdb{url = "http://user1:secret1@h/db/"}}, + {"user2", "secret2", #httpdb{url = "http://h/db/", headers = [Header]}}, + {"user3", "secret3", #httpdb{url = "http://h/db/", auth_props = [{~"basic", Basic}]}} + ] + ). + +httpdb_strip_creds_auth_plugins_test() -> + % We clean up various plugins as well + IAM = {[{~"iam", {[{~"api_key", ~"secret4"}]}}]}, + Other = {[{~"token", ~"secret5"}]}, + AuthProps = [{~"ibm", IAM}, {~"other", Other}], + HttpDb1 = httpdb_strip_creds(#httpdb{url = "http://h/db/", auth_props = AuthProps}), + ToStr = lists:flatten(io_lib:format("~p", [HttpDb1])), + ?assertEqual(nomatch, string:find(ToStr, "secret4")), + ?assertEqual(nomatch, string:find(ToStr, "secret5")), + ?assertEqual( + [ + {~"ibm", {[{~"iam", {[{~"api_key", ~"*****"}]}}]}}, + {~"other", {[{~"token", ~"*****"}]}} + ], + HttpDb1#httpdb.auth_props + ). + -endif.
