This is an automated email from the ASF dual-hosted git repository. willholley pushed a commit to branch wh/connect_to in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit eb3a970f183b6ebb78852b4212079f569de82384 Author: Will Holley <[email protected]> AuthorDate: Mon Apr 27 13:53:28 2026 +0100 feat: Add replicator DNS override support for outbound requests. This adds a feature to the CouchDB replicator to override the DNS target for specific host patterns (including wildcards) when making outbound requests. The use case is when requests need to be routed via a transparent SNI proxy e.g. for network egress monitoring. There is adds a new configuration option to specify the overrides: ``` [replicator] dns_overrides = host:target, host2:target ``` The replicator resolves the configured host patterns to the alternative connection targets while preserving the request URL host (applies to regular requests and session-auth requests). Note this depends on the `connect_to` option in ibrowse, which is a custom feature in the CouchDB ibrowse fork. --- rel/overlay/etc/default.ini | 3 + .../src/couch_replicator_auth_session.erl | 27 ++++++- src/couch_replicator/src/couch_replicator_dns.erl | 92 ++++++++++++++++++++++ .../src/couch_replicator_httpc.erl | 33 +++++++- .../test/eunit/couch_replicator_dns_tests.erl | 63 +++++++++++++++ 5 files changed, 216 insertions(+), 2 deletions(-) diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index bb017fb44..ff22bcf4f 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -707,6 +707,9 @@ partitioned||* = true ; Checkpoint interval ;checkpoint_interval = 30000 +; DNS overrides for replication requests +;dns_overrides = *.example.test:127.0.0.1 + ; Some socket options that might boost performance in some scenarios: ; {nodelay, boolean()} ; {sndbuf, integer()} diff --git a/src/couch_replicator/src/couch_replicator_auth_session.erl b/src/couch_replicator/src/couch_replicator_auth_session.erl index 93ddc834f..c66086926 100644 --- a/src/couch_replicator/src/couch_replicator_auth_session.erl +++ b/src/couch_replicator/src/couch_replicator_auth_session.erl @@ -311,11 +311,36 @@ refresh(#state{session_url = Url, user = User, pass = Pass} = State) -> {ok, string(), headers(), binary()} | {error, term()}. http_request(#state{httpdb_pool = Pool} = State, Url, Headers, Method, Body) -> Timeout = State#state.httpdb_timeout, - Opts = [ + + % Apply DNS override using connect_to ibrowse option + ParsedUrl = ibrowse_lib:parse_url(Url), + Host = element(3, ParsedUrl), % #url.host + Proto = element(2, ParsedUrl), % #url.protocol + {TargetHost, OriginalHost} = couch_replicator_dns:resolve_host(Host), + + Opts0 = [ {response_format, binary}, {inactivity_timeout, Timeout} | State#state.httpdb_ibrowse_options ], + + % Add connect_to ibrowse option if DNS override is active + Opts1 = case OriginalHost of + undefined -> + Opts0; + _ -> + couch_log:debug("DNS override for session: ~s -> ~s", [OriginalHost, TargetHost]), + [{connect_to, TargetHost} | Opts0] + end, + + % Add SNI for HTTPS with DNS override + Opts = case {Proto, OriginalHost} of + {https, OrigHost} when is_list(OrigHost) -> + couch_replicator_httpc:add_sni_option(Opts1, OrigHost); + _ -> + Opts1 + end, + {ok, Wrk} = couch_replicator_httpc_pool:get_worker(Pool), try Result = ibrowse:send_req_direct( diff --git a/src/couch_replicator/src/couch_replicator_dns.erl b/src/couch_replicator/src/couch_replicator_dns.erl new file mode 100644 index 000000000..985f2c25a --- /dev/null +++ b/src/couch_replicator/src/couch_replicator_dns.erl @@ -0,0 +1,92 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_replicator_dns). + +-export([ + resolve_host/1, + parse_config/1, + match_pattern/2, + get_overrides/0 +]). + +-type dns_override() :: {binary(), binary()}. + +-spec resolve_host(string()) -> {string(), string() | undefined}. +resolve_host(Host) -> + case find_override(unicode:characters_to_binary(Host), get_overrides()) of + {ok, Target} -> + {binary_to_list(Target), Host}; + not_found -> + {Host, undefined} + end. + +-spec get_overrides() -> [dns_override()]. +get_overrides() -> + case config:get("replicator", "dns_overrides", undefined) of + undefined -> + []; + ConfigStr -> + parse_config(ConfigStr) + end. + +-spec parse_config(string()) -> [dns_override()]. +parse_config(ConfigStr) -> + Entries = binary:split( + unicode:characters_to_binary(ConfigStr), <<",">>, [global, trim] + ), + lists:filtermap(fun parse_entry/1, Entries). + +parse_entry(<<>>) -> + false; +parse_entry(Entry0) -> + Entry = string:trim(Entry0), + case binary:split(Entry, <<":">>) of + [Pattern0, Target0] -> + Pattern = string:trim(Pattern0), + Target = string:trim(Target0), + case {Pattern, Target} of + {<<>>, _} -> invalid_entry(Entry); + {_, <<>>} -> invalid_entry(Entry); + _ -> {true, {Pattern, Target}} + end; + _ -> + invalid_entry(Entry) + end. + +invalid_entry(Entry) -> + couch_log:warning("Invalid dns_override entry: ~ts", [Entry]), + false. + +find_override(_Host, []) -> + not_found; +find_override(Host, [{Pattern, Target} | Rest]) -> + case match_pattern(Host, Pattern) of + true -> + {ok, Target}; + false -> + find_override(Host, Rest) + end. + +-spec match_pattern(binary() | string(), binary() | string()) -> boolean(). +match_pattern(Host0, Pattern0) -> + Host = unicode:characters_to_binary(Host0), + Pattern = unicode:characters_to_binary(Pattern0), + match_pattern_binary(Host, Pattern). + +match_pattern_binary(Host, <<"*", Suffix/binary>>) -> + HostSize = byte_size(Host), + SuffixSize = byte_size(Suffix), + HostSize >= SuffixSize andalso + binary:part(Host, HostSize - SuffixSize, SuffixSize) =:= Suffix; +match_pattern_binary(Host, Pattern) -> + Host =:= Pattern. \ No newline at end of file diff --git a/src/couch_replicator/src/couch_replicator_httpc.erl b/src/couch_replicator/src/couch_replicator_httpc.erl index fe81e65ea..f99c8edea 100644 --- a/src/couch_replicator/src/couch_replicator_httpc.erl +++ b/src/couch_replicator/src/couch_replicator_httpc.erl @@ -20,6 +20,7 @@ -export([send_req/3]). -export([stop_http_worker/0]). -export([full_url/2]). +-export([add_sni_option/2]). -import(couch_util, [ get_value/2, @@ -113,6 +114,10 @@ send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb0, Params) -> {Headers2, HttpDb} = couch_replicator_auth:update_headers(HttpDb0, Headers1), Url = full_url(HttpDb, Params), Body = get_value(body, Params, []), + + % Apply DNS override using connect_to ibrowse option + #url{host = Host, protocol = Protocol} = ibrowse_lib:parse_url(Url), + {TargetHost, OriginalHost} = couch_replicator_dns:resolve_host(Host), case get_value(path, Params) == "_changes" of true -> Timeout = infinity; @@ -131,7 +136,7 @@ send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb0, Params) -> {User, Pass} when is_list(User), is_list(Pass) -> [{basic_auth, {User, Pass}}] end, - IbrowseOptions = + IbrowseOptions0 = BasicAuthOpts ++ [ {response_format, binary}, @@ -142,6 +147,25 @@ send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb0, Params) -> HttpDb#httpdb.ibrowse_options ) ], + + % Add connect_to ibrowse option if DNS override is active + IbrowseOptions1 = case OriginalHost of + undefined -> + IbrowseOptions0; + _ -> + % Log DNS override for debugging + couch_log:debug("DNS override: ~s -> ~s", [OriginalHost, TargetHost]), + [{connect_to, TargetHost} | IbrowseOptions0] + end, + + % Add SNI for HTTPS with DNS override + IbrowseOptions = case {Protocol, OriginalHost} of + {https, OrigHost} when is_list(OrigHost) -> + add_sni_option(IbrowseOptions1, OrigHost); + _ -> + IbrowseOptions1 + end, + backoff_before_request(Worker, HttpDb, Params), Response = ibrowse:send_req_direct( Worker, Url, Headers2, Method, Body, IbrowseOptions, Timeout @@ -535,6 +559,13 @@ merge_headers(Headers1, Headers2) when is_list(Headers1), is_list(Headers2) -> Merged = mochiweb_headers:enter_from_list(Headers1 ++ Headers2, Empty), mochiweb_headers:to_list(Merged). +%% @private Add SNI to SSL options +add_sni_option(IbrowseOpts, Host) -> + SslOpts = proplists:get_value(ssl_options, IbrowseOpts, []), + SslOpts1 = [{server_name_indication, Host} | + proplists:delete(server_name_indication, SslOpts)], + lists:keystore(ssl_options, 1, IbrowseOpts, {ssl_options, SslOpts1}). + -ifdef(TEST). -include_lib("couch/include/couch_eunit.hrl"). diff --git a/src/couch_replicator/test/eunit/couch_replicator_dns_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_dns_tests.erl new file mode 100644 index 000000000..c260008d0 --- /dev/null +++ b/src/couch_replicator/test/eunit/couch_replicator_dns_tests.erl @@ -0,0 +1,63 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_replicator_dns_tests). + +-include_lib("couch/include/couch_eunit.hrl"). + +match_pattern_test_() -> + [ + ?_assert(couch_replicator_dns:match_pattern( + "account.example.test", "*.example.test")), + ?_assertNot(couch_replicator_dns:match_pattern( + "example.test", "*.example.test")), + ?_assert(couch_replicator_dns:match_pattern( + "exact.example.test", "exact.example.test")), + ?_assertNot(couch_replicator_dns:match_pattern( + "other.example.test", "exact.example.test")) + ]. + +parse_config_test_() -> + [ + ?_assertEqual( + 2, + length(couch_replicator_dns:parse_config( + "*.example.test:proxy.internal, exact.example.test:127.0.0.1" + )) + ), + ?_assertEqual([], couch_replicator_dns:parse_config("")) + ]. + +resolve_host_test_() -> + {setup, + fun() -> + meck:new(config, [passthrough]), + meck:expect(config, get, fun + ("replicator", "dns_overrides", _) -> + "*.example.test:egress.internal"; + (_, _, Default) -> + Default + end) + end, + fun(_) -> + meck:unload(config) + end, + [ + ?_assertEqual( + {"egress.internal", "account.example.test"}, + couch_replicator_dns:resolve_host("account.example.test") + ), + ?_assertEqual( + {"other.example.org", undefined}, + couch_replicator_dns:resolve_host("other.example.org") + ) + ]}.
