This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch replicator-heartbeat-seq in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit cc1d408601546df12b07ef025633e6420ef11c23 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Wed Oct 21 14:38:43 2026 -0400 Replicator support for heartbeat_seq change feed parameter This is a follow-up to heartbeat_seq PR #6165 where implementing heartbeat_seq handling in the replicator. Take advantage of the fact that unknown parameters are ignored by endpoints, so we always pass `heartbeat_seq=true`, and if we get any hearbeat rows we'll use them for checkpointing, and if we don't (an old source endpoint) we'll function as before. Issue: #6120 --- .../src/couch_replicator_api_wrap.erl | 42 ++++++++++++++--- .../src/couch_replicator_changes_reader.erl | 54 +++++++++++++++++++--- src/docs/src/replication/protocol.rst | 6 +++ 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/src/couch_replicator/src/couch_replicator_api_wrap.erl b/src/couch_replicator/src/couch_replicator_api_wrap.erl index c1ab912d4..57e15d2c4 100644 --- a/src/couch_replicator/src/couch_replicator_api_wrap.erl +++ b/src/couch_replicator/src/couch_replicator_api_wrap.erl @@ -547,12 +547,7 @@ changes_since( Timeout = max(1000, InactiveTimeout div 3), EncodedSeq = couch_replicator_utils:seq_encode(StartSeq), BaseQArgs = - case get_value(continuous, Options, false) of - false -> - [{"feed", "normal"}]; - true -> - [{"feed", "continuous"}] - end ++ + changes_feed_qargs(Options) ++ [ {"style", atom_to_list(Style)}, {"since", EncodedSeq}, @@ -669,6 +664,17 @@ maybe_add_changes_filter_q_args(BaseQS, Options) -> ] end. +changes_feed_qargs(Options) -> + Feed = + case get_value(continuous, Options, false) of + false -> "normal"; + true -> "continuous" + end, + % If the source supports heartbeat_seq=true we'll get and use {seq: ..., + % id: null, changes: []} heartbeat rows for checkpointing. If the source + % doesn't support it, it will ignore the parameter. + [{"feed", Feed}, {"heartbeat_seq", "true"}]. + parse_changes_feed(Options, UserFun, DataStreamFun) -> case get_value(continuous, Options, false) of true -> @@ -961,6 +967,9 @@ json_to_doc_info({Props}) -> case get_value(<<"changes">>, Props) of undefined -> {last_seq, get_value(<<"last_seq">>, Props)}; + [] -> + % Got a heartbeat row from heartbeat_seq=true param + {heartbeat_seq, get_value(~"seq", Props)}; Changes -> RevsInfo0 = lists:map( fun({Change}) -> @@ -1131,4 +1140,25 @@ normalize_http_db_test() -> ?assertEqual(Expected, normalize_db(HttpDb)), ?assertEqual(<<"local">>, normalize_db(<<"local">>)). +json_to_doc_info_test() -> + Seq = ~"42-abc", + ?assertEqual({last_seq, Seq}, json_to_doc_info({[{~"last_seq", Seq}]})), + ?assertEqual({last_seq, undefined}, json_to_doc_info({[{~"pending", 0}]})), + HB = {[{~"seq", Seq}, {~"id", null}, {~"changes", []}]}, + ?assertEqual({heartbeat_seq, Seq}, json_to_doc_info(HB)), + Row = + {[ + {~"seq", Seq}, + {~"id", ~"doc1"}, + {~"changes", [{[{~"rev", ~"1-abc"}]}]} + ]}, + ?assertEqual( + #doc_info{ + id = ~"doc1", + high_seq = Seq, + revs = [#rev_info{rev = {1, ~"abc"}}] + }, + json_to_doc_info(Row) + ). + -endif. diff --git a/src/couch_replicator/src/couch_replicator_changes_reader.erl b/src/couch_replicator/src/couch_replicator_changes_reader.erl index bb6733608..7ee121ced 100644 --- a/src/couch_replicator/src/couch_replicator_changes_reader.erl +++ b/src/couch_replicator/src/couch_replicator_changes_reader.erl @@ -132,18 +132,31 @@ process_change({last_seq, LS}, {_Parent, _, ChangesQueue, true = _Continuous}) - undefined -> get(last_seq); _ -> LS end, - OldSeq = get(last_seq), - if - Seq == OldSeq -> ok; - true -> ok = couch_work_queue:queue(ChangesQueue, {last_seq, Seq}) - end, - put(last_seq, Seq), + queue_last_seq(ChangesQueue, Seq), throw(recurse); process_change({last_seq, _}, _) -> % This clause is unreachable today, but let's plan ahead for the future % where we checkpoint against last_seq instead of the sequence of the last % change. The two can differ substantially in the case of a restrictive % filter. + ok; +process_change({heartbeat_seq, Seq}, {_Parent, _, ChangesQueue, _Continuous}) when + is_binary(Seq); is_integer(Seq) +-> + queue_last_seq(ChangesQueue, Seq); +process_change({heartbeat_seq, _}, _) -> + % We're being defensive here since we're in the replicator, and have to + % handle other, possibly non-ASF CouchDB implementations, or strange + % proxies. Here, we'd catch heartbeat objects without a seq or seq set to + % null, for example and we ignore them instead of crashing everything. + ok. + +queue_last_seq(ChangesQueue, Seq) -> + case get(last_seq) of + Seq -> ok; + _ -> ok = couch_work_queue:queue(ChangesQueue, {last_seq, Seq}) + end, + put(last_seq, Seq), ok. is_doc_id_too_long(IdLength) -> @@ -154,3 +167,32 @@ is_doc_id_too_long(IdLength) -> ConfigMax = list_to_integer(ConfigMaxStr), ConfigMax > 0 andalso IdLength > ConfigMax end. + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +heartbeat_seq_test() -> + {ok, Q} = couch_work_queue:new([{max_size, 100}, {max_items, 100}]), + Ctx = {self(), #httpdb{}, Q, true}, + put(last_seq, ~"1-a"), + % new sequence => last_seq + ?assertEqual(ok, process_change({heartbeat_seq, ~"2-b"}, Ctx)), + ?assertEqual(~"2-b", get(last_seq)), + ?assertEqual({ok, [{last_seq, ~"2-b"}]}, couch_work_queue:dequeue(Q)), + % dedup works + ?assertEqual(ok, process_change({heartbeat_seq, ~"2-b"}, Ctx)), + ?assertEqual(0, couch_work_queue:item_count(Q)), + % new seq is queued + ?assertEqual(ok, process_change({heartbeat_seq, ~"4-d"}, Ctx)), + ?assertEqual({ok, [{last_seq, ~"4-d"}]}, couch_work_queue:dequeue(Q)), + ?assertEqual(~"4-d", get(last_seq)), + % old replicators had integer sequences, we're maximally backwards + % compatible so test them too + ?assertEqual(ok, process_change({heartbeat_seq, 5}, Ctx)), + ?assertEqual({ok, [{last_seq, 5}]}, couch_work_queue:dequeue(Q)), + ?assertEqual(5, get(last_seq)), + erase(last_seq), + couch_work_queue:close(Q). + +-endif. diff --git a/src/docs/src/replication/protocol.rst b/src/docs/src/replication/protocol.rst index f361bc334..902af38bc 100644 --- a/src/docs/src/replication/protocol.rst +++ b/src/docs/src/replication/protocol.rst @@ -766,6 +766,12 @@ request. This request MUST be made with the following query parameters: period in *milliseconds*. The RECOMMENDED value by default is ``10000`` (10 seconds). +- The ``heartbeat_seq=true`` parameter MAY be used so that heartbeats will be + emitted as rows of the form ``{"seq": ..., "id": null, "changes": []}``. + Replicator SHOULD use its sequence to record a Checkpoint. This advances + Checkpoints when a filter rejects the Changes for a long time. A Source which + does not support the parameter SHOULD ignore it. + - If a startup Checkpoint was found during the Replication Logs comparison, the ``since`` query parameter MUST be passed with this value. In case of Full Replication it MAY be ``0`` (number zero) or
